From 3e1ced53ae1a787e242195c3738dcddf0da52912 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:55 +0530 Subject: [PATCH 01/46] drm/asahi: g15: m3: account PB backing and re-initialization in the growth ledger The PB ledger only knows the posted count. For backing that is allocated after a reply instead of before it, the ledger also has to know how many blocks are backed, grant a request only from those, and start over at a re-initialization without forgetting the backing. Add `backed` to the ledger. `accept()` now returns an answer: - a grant, clipped to the backed blocks above the posted count; - a refusal at the maximum (the refusal bound is unchanged); - a wait when no backed block is left: the request keeps its counter and can be accepted again once more is backed. `reinit()` returns the posted count to the initial size and keeps the backing; Pools::init() uses it instead of a new ledger. The growth chunks that back_pb() allocates before an answer are now 128 blocks (16 MiB) instead of 32. answer_request() copies the backed count into the ledger after back_pb(), so no request waits yet. Host tests (pools/growth.rs, run by the asahi-g15 selftest): the clamps and the producer bound over the constructor's boundary values with the heap backed, growth to the maximum and re-initialization, and grants clipped to the backing, with waits that keep the counter. The ledger is shared by the T6030 and the T6031; this is untested on the T6030. The growth harness follows the chunk size and the ledger reset. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit 94148052c842241c514d61cde07066cf0561901a) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/pools.rs | 18 +- drivers/gpu/drm/asahi/g15/pools/growth.rs | 195 +++++++++++++--------- tools/asahi/g15-pb-growth-tests.py | 54 +++--- 3 files changed, 161 insertions(+), 106 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index c635ed11c4a0c..561de12d405ef 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -7,7 +7,7 @@ //! //! - One 512 MiB PB heap at [`super::hw::client::PB_HEAP`] (4096 blocks of 128 KiB), backed on //! demand: the initial blocks when the GPU starts, then chunks of at most -//! [`growth::CHUNK_BLOCKS`] (4 MiB) as growth needs them ([`Firmware::back_pb`]), never freed +//! [`growth::CHUNK_BLOCKS`] (16 MiB) as growth needs them ([`Firmware::back_pb`]), never freed //! or shrunk. The backing ([`PbChunk`]) is mapped only into user VMs, at the fixed client GVA, //! each VM's before its prefixes are published ([`Firmware::map_pb`]); nothing names its pages //! through the host window or context 0 (the records hold client GVAs only). Initial @@ -377,10 +377,11 @@ impl Pools { true, )?; // Backing for the initial blocks; growth adds the rest. - let ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let mut ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); let mut chunks = KVec::with_capacity(growth::max_chunks(ledger.initial, ledger.max), GFP_KERNEL)?; chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?; + ledger.backed = ledger.initial; Ok(Self { dev: dev.as_ref().as_ref().into(), drm: dev.into(), @@ -451,8 +452,9 @@ impl Pools { // PB: 16384 pages of 0x8000 in 4096 blocks of 0x20000; with growth the first initial // blocks of them, at most 4095 posted, and page limits at the firmware's growth maximum. - // In every profile the block list declares its real 4096-record capacity. - self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + // The backing is kept. In every profile the block list declares its real 4096-record + // capacity. + self.ledger.reinit(); self.exhausted_logged = false; if tuning::PB_GROW { // C1 names the initial pages again. @@ -766,7 +768,13 @@ impl Firmware { // made so far stays for the next request. let end = self.pools.ledger.posted + self.pools.ledger.next_count(); self.back_pb(end, ctx)?; - let (first, n) = self.pools.ledger.accept(request); + self.pools.ledger.backed = self.pools.backed(); + let (first, n) = match self.pools.ledger.accept(request) { + growth::Answer::Grant { first, count } => (first, count), + growth::Answer::Refuse => (self.pools.ledger.posted, 0), + // Backed above: no request waits here. + growth::Answer::Wait => return Err(ENOSPC), + }; let heap = client::PB_HEAP; for b in first..first + n { self.pools.pb_blocks.w64( diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 20b298e641459..0ce3696ebac55 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -12,6 +12,12 @@ //! its recovery path then apply, without a guarantee that they end the stall. A request from //! before a halt (epoch) or not matching a published render prefix is left unanswered. //! +//! The heap's VA range covers [`MAX_BLOCKS`] blocks; its RAM comes in chunks from the heap +//! base, which the ledger counts as `backed`. A request is granted from the backed blocks above +//! `posted`; a request that finds no backed block waits ([`Answer::Wait`]) and keeps its +//! counter. A re-initialization ([`Ledger::reinit`]) returns the posted count to the initial +//! size and keeps the backing. +//! //! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. use crate::g15::fw::buffer::{grow_request, BLOCK_LIST_RECORDS, GROW_STEP, MAX_BLOCKS, MIN_BLOCKS}; @@ -26,6 +32,8 @@ pub(crate) struct Ledger { pub(crate) max: u32, /// Blocks posted to the block list, cumulative (the block_ctl+4 producer is this modulo K). pub(crate) posted: u32, + /// Blocks with RAM behind them, from the heap base. + pub(crate) backed: u32, /// The next expected request counter. pub(crate) next_request: u32, /// A refusal at the maximum was logged since the last initialization. @@ -41,9 +49,20 @@ pub(crate) struct Ledger { /// Refusals answered at the maximum before requests are left to the deadline. pub(crate) const MAX_REFUSALS: u32 = 2; -/// Blocks of each PB backing chunk after the initial one (4 MiB): growth backs at most this -/// much at a time while the firmware waits for its answer. -pub(crate) const CHUNK_BLOCKS: u32 = 32; +/// Blocks of each PB backing chunk (16 MiB). +pub(crate) const CHUNK_BLOCKS: u32 = 128; + +/// The answer to an admitted growth request. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Answer { + /// Post blocks `[first, first + count)` (`count` at most [`GROW_STEP`]) and reply grown. + Grant { first: u32, count: u32 }, + /// Reply refused: the maximum is posted. + Refuse, + /// No backed block above the posted count: leave the request queued until a chunk is + /// backed. + Wait, +} /// The backing chunk that follows `backed` blocks of a ledger with maximum `max`: its first block /// and size, at most [`CHUNK_BLOCKS`] and never past `max`; none from `max` on. @@ -85,17 +104,32 @@ impl Ledger { initial, max, posted: initial, + backed: 0, next_request: 0, refusal_logged: false, refusals: 0, } } + /// A re-initialization: the initial blocks are posted again, the request counter and the + /// refusals start over, the backing stays. + pub(crate) const fn reinit(&mut self) { + self.posted = self.initial; + self.next_request = 0; + self.refusal_logged = false; + self.refusals = 0; + } + /// The live block_ctl+4 producer. pub(crate) const fn producer(&self) -> u32 { self.posted % BLOCK_LIST_RECORDS } + /// Backed blocks above the posted count. + pub(crate) const fn spare(&self) -> u32 { + self.backed.saturating_sub(self.posted) + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -114,21 +148,30 @@ impl Ledger { && request_epoch(request) == epoch } - /// The blocks the next accepted request posts: at most [`GROW_STEP`], none at the maximum. + /// The blocks the next accepted request posts with the backing it needs: at most + /// [`GROW_STEP`], none at the maximum. pub(crate) fn next_count(&self) -> u32 { GROW_STEP.min(self.max - self.posted) } - /// Accepts an admitted request: the next expected counter follows it, and the returned - /// blocks `[first, first + count)` are to be written to the block list, `count` 0 meaning a - /// refusal at the maximum. - pub(crate) fn accept(&mut self, request: &[u32; 14]) -> (u32, u32) { + /// Accepts an admitted request. Granted (at most [`GROW_STEP`] blocks, and no more than are + /// backed) or refused at the maximum, the next expected counter follows it. A request that + /// finds no backed block waits: its counter is not consumed, and it is accepted again once a + /// chunk is backed. + pub(crate) fn accept(&mut self, request: &[u32; 14]) -> Answer { + let room = self.max - self.posted; + if room != 0 && self.spare() == 0 { + return Answer::Wait; + } self.next_request = request[grow_request::COUNTER].wrapping_add(1); - let count = self.next_count(); - if count == 0 { + if room == 0 { self.refusals += 1; + return Answer::Refuse; + } + Answer::Grant { + first: self.posted, + count: GROW_STEP.min(room).min(self.spare()), } - (self.posted, count) } /// Records `count` blocks as posted after their records were written and returns the new @@ -185,12 +228,19 @@ mod tests { q } + /// A ledger with the heap backed whole. + fn backed(grow: bool, blocks: u32, max_blocks: u32) -> Ledger { + let mut l = Ledger::new(grow, blocks, max_blocks); + l.backed = MAX_BLOCKS; + l + } + #[test] fn clamps() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); assert_eq!( - (l.initial, l.max, l.posted, l.next_request), - (1024, 2048, 1024, 0) + (l.initial, l.max, l.posted, l.backed, l.next_request), + (1024, 2048, 1024, 4096, 0) ); assert_eq!(l.producer(), 1024); assert_eq!(Ledger::new(true, 10, 20).initial, 64); @@ -198,6 +248,7 @@ mod tests { assert_eq!(Ledger::new(true, 5000, 100).initial, 4095); assert_eq!(Ledger::new(true, 512, 1024).max, 1024); assert_eq!(Ledger::new(true, 512, 100).max, 512); + assert_eq!(Ledger::new(true, 512, 100).backed, 0); let fixed = Ledger::new(false, 512, 1024); assert_eq!((fixed.initial, fixed.max, fixed.posted), (4095, 4095, 4095)); // The values the driver passes (tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS) are kept. @@ -235,18 +286,19 @@ mod tests { for grow in [true, false] { for blocks in values { for max_blocks in values { - let mut l = Ledger::new(grow, blocks, max_blocks); + let mut l = backed(grow, blocks, max_blocks); assert!(MIN_BLOCKS <= l.initial && l.initial <= l.max && l.max <= MAX_POSTED); assert_eq!(l.posted, l.initial); assert!((1..=MAX_POSTED).contains(&l.producer())); // Every producer of a growth to the maximum stays in 1..=4095. let mut counter = 0; loop { - let (first, n) = l.accept(&request(5, counter, 1)); + let (first, n) = match l.accept(&request(5, counter, 1)) { + Answer::Grant { first, count } => (first, count), + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert_eq!(first, l.posted); - if n == 0 { - break; - } let producer = l.post(n); assert!((1..=MAX_POSTED).contains(&producer)); assert_eq!(producer, l.posted); @@ -261,7 +313,7 @@ mod tests { #[test] fn admission() { - let l = Ledger::new(true, 1024, 2048); + let l = backed(true, 1024, 2048); let q = request(5, 0, 3); assert!(l.admits(&q, true, true, 3)); assert!(!l.admits(&q, false, true, 3)); @@ -283,17 +335,20 @@ mod tests { #[test] fn growth_to_the_maximum() { // 512 initial and 1024 maximum blocks. - let mut l = Ledger::new(true, 512, 1024); + let mut l = backed(true, 512, 1024); let mut grown = 0; let mut counter = 0; loop { let q = request(5, counter, 1); assert!(l.admits(&q, true, true, 1)); - let (first, n) = l.accept(&q); - assert_eq!(first, 512 + grown); - if n == 0 { - break; - } + let n = match l.accept(&q) { + Answer::Grant { first, count } => { + assert_eq!(first, 512 + grown); + count + } + Answer::Refuse => break, + Answer::Wait => panic!("backed whole"), + }; assert!(n <= 10); assert_eq!(l.post(n), (512 + grown + n) % 4096); grown += n; @@ -310,69 +365,57 @@ mod tests { assert_eq!(l.refusals, 1); assert!(!l.exhausted()); let q = request(5, counter + 3, 1); - assert_eq!(l.accept(&q), (1024, 0)); + assert_eq!(l.accept(&q), Answer::Refuse); assert!(l.exhausted()); assert!(!l.admits(&request(5, counter + 4, 1), true, true, 1)); - // A fresh ledger starts over. + // A fresh ledger starts over, and so does a re-initialized one (backing kept). assert!(!Ledger::new(true, 512, 1024).exhausted()); - } - - /// Backing for every ledger the constructor makes, the driver's included: the chunks from - /// the initial one are contiguous, at most [`CHUNK_BLOCKS`] each, end at the maximum and - /// number at most [`max_chunks`]; growth with the backing made first never posts a block - /// past it. - #[test] - fn backing_chunks() { - assert_eq!(max_chunks(1024, 4095), 1 + 96); - assert_eq!(max_chunks(MIN_BLOCKS, MAX_POSTED), 1 + 126); - assert_eq!(max_chunks(MAX_POSTED, MAX_POSTED), 1); - assert_eq!(next_chunk(1024, 4095), Some((1024, 32))); - assert_eq!(next_chunk(4064, 4095), Some((4064, 31))); - assert_eq!(next_chunk(4095, 4095), None); - assert_eq!(next_chunk(4096, 4095), None); - let values = [ - 0, 1, MIN_BLOCKS, 65, 1023, 1024, 1025, 3072, 4094, MAX_POSTED, 5000, - ]; - for grow in [true, false] { - for blocks in values { - for max_blocks in values { - let mut l = Ledger::new(grow, blocks, max_blocks); - let (mut backed, mut chunks) = (l.initial, 1); - let mut counter = 0; - loop { - let end = l.posted + l.next_count(); - while backed < end { - let (first, n) = next_chunk(backed, l.max).unwrap(); - assert_eq!(first, backed); - assert!(n >= 1 && n <= CHUNK_BLOCKS); - backed += n; - chunks += 1; - } - let (first, n) = l.accept(&request(5, counter, 1)); - assert!(first + n <= backed); - if n == 0 { - break; - } - l.post(n); - counter += 1; - } - assert_eq!((l.posted, backed), (l.max, l.max)); - assert!(chunks <= max_chunks(l.initial, l.max)); - assert_eq!(next_chunk(backed, l.max), None); - } - } - } + l.reinit(); + assert_eq!( + (l.posted, l.next_request, l.refusals, l.backed), + (512, 0, 0, 4096) + ); + assert!(!l.exhausted() && !l.refusal_logged); } #[test] fn producer_wraps_at_k() { - let mut l = Ledger::new(true, 2048, 2048); + let mut l = backed(true, 2048, 2048); assert_eq!(l.producer(), 2048); l.posted = 4095; assert_eq!(l.post(1), 0); assert_eq!(l.post(10), 10); } + #[test] + fn grants_only_backed_blocks() { + let mut l = Ledger::new(true, 64, 4095); + // Nothing backed above the initial posting: a request waits and keeps its counter. + l.backed = 64; + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + // A grant is clipped to the backed blocks; the next request waits again. + l.backed = 70; + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 6 + } + ); + l.post(6); + assert_eq!(l.next_request, 1); + assert_eq!(l.accept(&request(3, 1, 1)), Answer::Wait); + // At the maximum the refusal does not wait for backing. + l.posted = l.max; + l.backed = l.max; + assert_eq!(l.accept(&request(3, 1, 1)), Answer::Refuse); + // A re-initialization keeps the backing. + l.reinit(); + assert_eq!((l.posted, l.backed, l.next_request), (64, 4095, 0)); + } + #[test] fn logging() { let mut l = Ledger::new(true, 64, 64); diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 12ee4f16a4955..67fa40aab75f3 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -27,7 +27,7 @@ exactly is not; - invalid requests (type, slot, stale counter, epoch, no live render) get no reply, and requests rejected so allocate nothing; -- the backing: the initial blocks only when the GPU starts; chunks of at most 4 MiB, allocated +- the backing: the initial blocks only when the GPU starts; chunks of at most 16 MiB, allocated and held by the running VM before any record or producer names their blocks, never past the maximum, at most growth::max_chunks(); a failed allocation, alias map or append posts and answers nothing, leaves the request counter and the VM's count alone and drops what is not @@ -98,12 +98,12 @@ }''' # Pools::new's PB backing, run verbatim by the model. -POOLS_NEW_BACKING = (' // Backing for the initial blocks; growth adds the rest.\n', ' chunks.push(') +POOLS_NEW_BACKING = (' // Backing for the initial blocks; growth adds the rest.\n', ' ledger.backed = ') # The compile-time proof that a PB alias drop is a plain unmap. PB_LEAF_ASSERT = 'static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent());' # The two statements of Pools::init() that start a new ledger. -LEDGER_RESET = ['self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS);', +LEDGER_RESET = ['self.ledger.reinit();', 'self.exhausted_logged = false;'] MUTATIONS = { @@ -140,12 +140,14 @@ 'let nb = clamp(blocks, MIN_BLOCKS, MAX_BLOCKS);\n' ' (nb, clamp(max_blocks, nb, MAX_BLOCKS))\n' ' } else {\n (MAX_BLOCKS, MAX_BLOCKS)'), - # Post blocks without backing them first. - 'grant-without-backing': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', - ' let (first, n) = self.pools.ledger.accept(request);'), + # Post blocks without backing them first (the ledger told the whole heap is backed). + 'grant-without-backing': (POOLS, ' self.back_pb(end, ctx)?;\n self.pools.ledger.backed = self.pools.backed();\n', + ' self.pools.ledger.backed = buffer::MAX_BLOCKS;\n'), # Back the blocks only after the request is accepted. - 'back-after-accept': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', - ' let (first, n) = self.pools.ledger.accept(request);\n self.back_pb(end, ctx)?;'), + 'back-after-accept': [(POOLS, ' self.back_pb(end, ctx)?;\n self.pools.ledger.backed = self.pools.backed();\n', + ' self.pools.ledger.backed = buffer::MAX_BLOCKS;\n'), + (POOLS, ' growth::Answer::Wait => return Err(ENOSPC),\n };\n', + ' growth::Answer::Wait => return Err(ENOSPC),\n };\n self.back_pb(end, ctx)?;\n')], # Count a catch-up alias before the VM holds it. 'catch-up-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n mapped += 1;\n' ' vm.pb_chunks.store(mapped, Ordering::Relaxed);', @@ -174,8 +176,8 @@ # Back the whole heap when the GPU starts. 'initial-backs-whole-heap': (POOLS, 'chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?;', 'chunks.push(PbChunk::new(dev, 0, buffer::MAX_BLOCKS)?, GFP_KERNEL)?;'), - # Growth chunks of 8 MiB. - 'chunks-above-4mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 32;', 'pub(crate) const CHUNK_BLOCKS: u32 = 64;'), + # Growth chunks of 32 MiB. + 'chunks-above-16mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 128;', 'pub(crate) const CHUNK_BLOCKS: u32 = 256;'), # A firmware-visible cached PB leaf (its drop would ask the firmware to flush). 'pb-leaf-fw-visible': (HW, 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c83;', 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c03;'), @@ -204,7 +206,7 @@ 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), 'no-same-vm-checks': ('grant_refuses_flights_of_two_vms', 'grant with flights of two VMs'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), - 'chunks-above-4mib': ('growth_backs_before_posting', 'a growth chunk above 4 MiB'), + 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), 'pb-leaf-fw-visible': ('pb_leaf_is_gpu_only', 'PB leaf not GPU-only'), } assert set(INTENDED) == set(MUTATIONS) @@ -980,7 +982,9 @@ def consts(text, names): assert!(log.iter().all(|e| matches!(e, Ev::Log("warn", _))), "{log:?}"); assert_eq!(log.len(), 4); assert_eq!(f.pools.stats.bad, 4); - assert_eq!(f.pools.ledger, Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS)); + let mut fresh = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + fresh.backed = fresh.initial; + assert_eq!(f.pools.ledger, fresh); // No render published: not live. f.renders[0] = 0; f.events.grow.0.push_back(good); @@ -1074,7 +1078,7 @@ def consts(text, names): assert_eq!(log, vec![Ev::Alloc(1, initial as usize * BLOCK_BYTES as usize)], "startup allocation"); assert_eq!((f.pools.backed(), f.pools.chunk_count()), (initial, 1)); assert_eq!(f.pools.chunk_capacity(), max_chunks(initial, max)); - assert_eq!(max_chunks(initial, max), 97); + assert_eq!(max_chunks(initial, max), 25); // The first prefix of a VM maps the backing, nothing else. let vm = VmState::new(7); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); @@ -1109,14 +1113,14 @@ def consts(text, names): let log = take(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); assert_posted_backed(&log, 5, initial as u64); - // Chunks of at most 32 blocks, allocated only as needed: up to the maximum, no further. + // Chunks of at most 128 blocks, allocated only as needed: up to the maximum, no further. let allocs: Vec = log .iter() .filter_map(|e| if let Ev::Alloc(_, b) = e { Some(*b) } else { None }) .collect(); assert_eq!(allocs.len() + 1, max_chunks(initial, max)); - assert!(allocs.iter().all(|b| *b <= 4 << 20), "a growth chunk above 4 MiB"); - assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 4 << 20); + assert!(allocs.iter().all(|b| *b <= 16 << 20), "a growth chunk above 16 MiB"); + assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 16 << 20); assert_eq!(f.pools.backed(), max); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); assert_cursor(&f.engine.flights[0].job.vm, f.pools.chunk_count()); @@ -1187,7 +1191,7 @@ def consts(text, names): let a = f.engine.flights[0].job.vm.clone(); grow_to(&mut f, 1100); take(); - assert_eq!(f.pools.chunk_count(), 4); + assert_eq!(f.pools.chunk_count(), 2); // A's work retires; a new VM runs next: it maps every chunk first. f.engine.flights.clear(); let b = VmState::new(6); @@ -1195,13 +1199,13 @@ def consts(text, names): f.fw.ctx = 6; f.map_pb(&b).unwrap(); let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Append(6, _))), 4); + assert_eq!(count(&log, |e| matches!(e, Ev::Append(6, _))), 2); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 0); - assert_cursor(&b, 4); + assert_cursor(&b, 2); // B grows the backing; A, behind, catches up when it runs again. grow_to(&mut f, 1200); let log = take(); - assert_posted_backed(&log, 6, 1120); + assert_posted_backed(&log, 6, 1152); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 0); f.engine.flights.clear(); f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); @@ -1236,7 +1240,7 @@ def consts(text, names): #[test] fn partial_catch_up_counts_only_held_aliases() { let mut f = Firmware::new(); - grow_to(&mut f, 1100); + grow_to(&mut f, 1450); f.engine.flights.clear(); let b = VmState::new(6); f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); @@ -1246,17 +1250,17 @@ def consts(text, names): crate::MAP_OK_BEFORE_FAIL.with(|c| c.set(2)); FAIL_MAP.with(|c| c.set(1)); assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_cursor(&b, 4); + assert_cursor(&b, 5); assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2); // The third alias is made but cannot be held: dropped, still two. FAIL_APPEND.with(|c| c.set(1)); assert_eq!(f.map_pb(&b), Err(ENOMEM)); assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2, "count after a failed append"); assert!(take().iter().any(|e| matches!(e, Ev::Unmap(6, _)))); - assert_cursor(&b, 4); + assert_cursor(&b, 5); f.map_pb(&b).unwrap(); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 4); - assert_cursor(&b, 4); + assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 5); + assert_cursor(&b, 5); } /// The ledger reset of Pools::init(), not a full recovery. From 3a7dea21d89e9dee6c8b076c710f7de81d5ddc85 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:56 +0530 Subject: [PATCH 02/46] drm/asahi: g15: m3: back the PB in 16 MiB chunks after the reply, with a reserve back_pb() allocates and maps the backing of a growth request before the answer, while the firmware waits for it. The allocation has no time bound (shmem pages with __GFP_RETRY_MAYFAIL), and a failure leaves the request unanswered until the host deadline. Keep one chunk of backed blocks in reserve above the posted count and back the next chunk after the replies of a poll pass: - the heap is backed in chunks of 128 blocks (16 MiB), each a gpu_backing() object at its fixed client GVA (chunk_gva()); - when the GPU starts, the chunks of the initial posting and one reserve chunk are backed (9 with the initial 1024 blocks); - answer_request() grants only from backed blocks; while fewer than 128 remain above the posted count, refill_backing() backs chunks at the end of answer_growth(), after every reply of the pass; - with the reserve dry (an allocation failed), a request waits: it goes back to the head of the queue with its counter and is answered by a later pass once a chunk is backed; - a pending re-initialization backs nothing while no render is published; a render published before it (the flights behind a fragment kill) is still backed, or its requests would wait until the deadline. A compute-only flight gets none: it cannot ask for growth. Mapping stays per VM and before the VM can reach a block. A VM maps every chunk it lacks at the start of its prefix's publication (map_pb(), with its one-VM check), and a chunk backed by a refill is mapped into the VM of the flights before the ledger counts it. If that mapping fails, the chunk is dropped again and the next pass retries. The aliases are held in the VM's own heap list (VmState::map_heap()), which keeps the object of each alias. A chunk whose object changed is unmapped and mapped again, and aliases past the backing are dropped. back_pb(), running_vm() and the per-VM count are removed: no grant can name a block that the running VM has not mapped. This applies to the T6030 and the T6031, which share this code; it is untested on the T6030. The debugfs pb: line appends waits and refills. The growth harness now runs answer_growth(), answer_request(), map_pb(), refill_backing() and the VM heap-mapping bodies. Its new cases: - the start backing; - allocation only after the pass's replies; - a dry reserve that makes a request wait and keeps its order; - a failed refill map that drops the chunk; - a pending re-initialization with no flight, with a compute-only flight, and with a render published before it; - catch-up at publication, including a partial one. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit 7eae5b05328ca1d86ac1e770a5170829a12f4ec6) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/pools.rs | 345 +++++++------ drivers/gpu/drm/asahi/g15/pools/growth.rs | 163 ++++-- drivers/gpu/drm/asahi/g15/vm.rs | 91 +++- tools/asahi/g15-pb-growth-tests.py | 603 ++++++++++++---------- 4 files changed, 739 insertions(+), 463 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index 561de12d405ef..15204f6091613 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -6,16 +6,21 @@ //! (host-tested). //! //! - One 512 MiB PB heap at [`super::hw::client::PB_HEAP`] (4096 blocks of 128 KiB), backed on -//! demand: the initial blocks when the GPU starts, then chunks of at most -//! [`growth::CHUNK_BLOCKS`] (16 MiB) as growth needs them ([`Firmware::back_pb`]), never freed -//! or shrunk. The backing ([`PbChunk`]) is mapped only into user VMs, at the fixed client GVA, -//! each VM's before its prefixes are published ([`Firmware::map_pb`]); nothing names its pages -//! through the host window or context 0 (the records hold client GVAs only). Initial +//! demand in chunks of [`growth::CHUNK_BLOCKS`] blocks (16 MiB) from the heap base: when the +//! GPU starts, the chunks of the initial posting and one reserve chunk +//! ([`growth::Ledger::chunks_for`]). The backing ([`PbChunk`]) is mapped only into user VMs, at +//! the fixed client GVA: each VM's before its prefixes are published ([`Firmware::map_pb`]), +//! and a chunk backed later also into the VM of the flights; nothing names its pages through +//! the host window or context 0 (the records hold client GVAs only). Initial //! [`super::hw::tuning::PB_BLOCKS`] (1024), at most [`super::hw::tuning::PB_MAX_BLOCKS`] //! (4095) posted, so the producer never returns to 0; the firmware is given //! [`Pools::soft_max`] (default 3072) as its growth maximum. Truthful K = 4096 in every //! profile (PB reconstruction, also in recovery, synchronizes 8 * K bytes of the block list: //! K = 0x26cd overran the 0x8000-byte list and crashed the second recovery). +//! - Backing: a request is granted from the backed blocks above the posted count; after the +//! replies of a pass, while fewer than a reserve chunk remain, [`Firmware::refill_backing`] +//! backs the next chunk, so a reply never waits for the allocator. Only a reserve that ran dry +//! makes a request wait: it stays queued, the pass backs a chunk and the next pass answers it. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -41,7 +46,6 @@ mod growth; use core::fmt; -use core::sync::atomic::Ordering; use kernel::{ device, prelude::*, @@ -58,7 +62,7 @@ use super::Firmware; use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{gem, mmu}; -pub(crate) use growth::Ledger; +pub(crate) use growth::{Answer, Ledger}; static_assert!(buffer::MAX_BLOCKS == sizes::PB_BLOCKS); static_assert!(buffer::MAX_PAGES == sizes::PB_PAGES); @@ -75,47 +79,36 @@ static_assert!(tuning::PB_BLOCKS <= tuning::PB_MAX_BLOCKS); // The PB alias is GPU-only (not cached and firmware-visible): dropping one is a plain unmap, // with no firmware cache flush (`KernelMapping::drop`). static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent()); +static_assert!(growth::MAX_CHUNKS * growth::CHUNK_BLOCKS == buffer::MAX_BLOCKS); -/// Physical backing of PB blocks `[first, first + blocks)`: a zeroed GEM kernel object, pinned -/// and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at the blocks' fixed -/// client GVA ([`Firmware::map_pb`]). Retained until reboot, like every firmware object; each -/// alias also holds the object. +/// Bytes of one backing chunk. +pub(crate) const CHUNK_BYTES: usize = growth::CHUNK_BLOCKS as usize * buffer::BLOCK_BYTES as usize; + +/// Client GVA of backing chunk `i`. +pub(crate) const fn chunk_gva(i: usize) -> u64 { + client::PB_HEAP + (i * CHUNK_BYTES) as u64 +} + +/// Physical backing of one chunk of PB blocks: a zeroed GEM kernel object of [`CHUNK_BYTES`], +/// pinned and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at its fixed +/// client GVA ([`chunk_gva`], [`VmState::map_heap`]). Each alias also holds the object. struct PbChunk { - first: u32, - blocks: u32, _pages: HeldPages, backing: gem::ObjectRef, } impl PbChunk { - fn new(dev: &AsahiDevice, first: u32, blocks: u32) -> Result { - if blocks == 0 || first.checked_add(blocks).ok_or(EINVAL)? > buffer::MAX_BLOCKS { - return Err(EINVAL); - } - let (backing, pages) = - object::gpu_backing(dev, blocks as usize * buffer::BLOCK_BYTES as usize)?; + fn new(dev: &AsahiDevice) -> Result { + let (backing, pages) = object::gpu_backing(dev, CHUNK_BYTES)?; Ok(Self { - first, - blocks, _pages: pages, backing, }) } - /// The block after the chunk's last. - fn end(&self) -> u32 { - self.first + self.blocks - } - - /// Maps the whole chunk, exactly its object's size, at its fixed client GVA in `vm`. - fn map(&self, vm: &mmu::Vm) -> Result { - vm.map_at( - client::PB_HEAP + self.first as u64 * buffer::BLOCK_BYTES, - self.blocks as usize * buffer::BLOCK_BYTES as usize, - &self.backing.gem, - mmu::Prot::from_pte(pte::LEAF_CACHED), - false, - ) + /// The GEM object, for the VM mappings. + fn gem(&self) -> ARef { + self.backing.gem.clone() } } @@ -161,6 +154,10 @@ pub(crate) struct PbStats { pub(crate) grown: u64, pub(crate) refused: u64, pub(crate) bad: u64, + /// Requests that waited a pass for backing. + pub(crate) waits: u64, + /// Chunks backed after the GPU started. + pub(crate) refills: u64, } /// Render allocator and PB state: the allocators, the PB ledger and the spare set. @@ -179,9 +176,8 @@ pub(crate) struct Pools { spare_ready: bool, pb_pages: Arc, pb_blocks: Arc, - /// The PB backing, contiguous from block 0, in block order: the initial blocks, then the - /// growth chunks. Its capacity, reserved up front, is the most the ledger can need - /// ([`growth::max_chunks`]). + /// The PB backing chunks, from the heap base (chunk `i` at [`chunk_gva`]`(i)`). Its + /// capacity, reserved up front, is the whole heap's [`growth::MAX_CHUNKS`]. chunks: KVec, /// The PB growth ledger since the last (re)initialization. pub(crate) ledger: Ledger, @@ -376,12 +372,20 @@ impl Pools { 0, true, )?; - // Backing for the initial blocks; growth adds the rest. + // Backing for the initial blocks and the reserve; growth adds the rest. let mut ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); - let mut chunks = - KVec::with_capacity(growth::max_chunks(ledger.initial, ledger.max), GFP_KERNEL)?; - chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?; - ledger.backed = ledger.initial; + let mut chunks = KVec::with_capacity(growth::MAX_CHUNKS as usize, GFP_KERNEL)?; + for _ in 0..Ledger::chunks_for(ledger.initial) { + chunks.push(PbChunk::new(dev)?, GFP_KERNEL)?; + ledger.back_chunk(); + } + dev_info!( + dev.as_ref(), + "G15: PB heap: {} blocks posted at start, {} chunks of {} MiB backed\n", + ledger.initial, + chunks.len(), + CHUNK_BYTES >> 20 + ); Ok(Self { dev: dev.as_ref().as_ref().into(), drm: dev.into(), @@ -472,7 +476,7 @@ impl Pools { } let heap = client::PB_HEAP; let initial = self.ledger.initial; - if initial > self.backed() { + if initial > self.ledger.backed { return Err(EINVAL); } write_records( @@ -526,133 +530,130 @@ impl Pools { super::params::pb_max_blocks().clamp(self.ledger.initial, self.ledger.max) } - /// The blocks with backing, from block 0. - pub(crate) fn backed(&self) -> u32 { - self.chunks.last().map_or(0, PbChunk::end) + /// Maps the backed chunks `vm` does not hold yet ([`VmState::map_heap`]). + fn map_heap(&self, vm: &VmState) -> Result { + vm.map_heap(self.chunks.iter().map(PbChunk::gem)) + } + + /// Backs one more chunk; the caller maps it where needed before the ledger posts from it. + fn back_chunk(&mut self) -> Result> { + if self.chunks.len() >= growth::MAX_CHUNKS as usize { + return Err(ENOSPC); + } + let chunk = PbChunk::new(&self.drm)?; + let gem = chunk.gem(); + self.chunks.push(chunk, GFP_KERNEL)?; + Ok(gem) } /// The debugfs line `pb: blocks initial/posted/firmware-max/max backed blocks/chunks - /// requests grown refused bad`. + /// requests grown refused bad waits refills` (new keys are appended). pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { writeln!( f, - "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {}", + "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {} waits {} refills {}", self.ledger.initial, self.ledger.posted, self.soft_max(), self.ledger.max, - self.backed(), + self.ledger.backed, self.chunks.len(), self.stats.requests, self.stats.grown, self.stats.refused, - self.stats.bad + self.stats.bad, + self.stats.waits, + self.stats.refills ) } } impl Firmware { - /// Maps the PB backing into `vm` up to its last chunk, before anything of a prefix of the - /// VM is published (the start of `publish_prefix_into`). Render work runs in one VM at a - /// time (`Engine::publish`), so the VM about to run is the only one through which the GPU - /// can reach a posted block next; a VM created or idle while the backing grew catches up - /// here, also a closed VM whose accepted jobs still run. + /// Maps the PB backing into `vm`, before anything of a prefix of the VM is published (the + /// start of `publish_prefix_into`). Render work runs in one VM at a time + /// (`Engine::publish`), so the VM about to run is the only one through which the GPU can + /// reach a posted block next; a VM created or idle while the backing grew catches up here, + /// also a closed VM whose accepted jobs still run. A chunk is mapped only once + /// ([`VmState::map_heap`]); one that fails leaves the chunks mapped before it, and the + /// prefix is not published. The leaf is GPU-only, so dropping an alias is a plain unmap. /// - /// Each chunk's alias joins the VM's driver mappings, held until the VM is torn down, and - /// the VM's count moves past the chunk only then. When that fails, the alias just made is - /// dropped again: nothing of the VM runs before its count covers the chunk, and the leaf is - /// GPU-only, so the drop is a plain unmap. The chunks mapped before stay counted. - /// - /// The one-VM rule is checked first: a flight of another VM, or a count past the chunks, - /// fails with EPROTO before anything is mapped. + /// The one-VM rule is checked first: a flight of another VM fails with EPROTO before + /// anything is mapped. pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result { - let mut mapped = vm.pb_chunks.load(Ordering::Relaxed); - if mapped > self.pools.chunks.len() - || self - .engine - .flights - .iter() - .any(|f| !core::ptr::eq(&*f.job.vm, vm)) - { + let others = self + .engine + .flights + .iter() + .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) + .count(); + if others != 0 { dev_err!( self.dev(), - "PB backing: ctx {} maps {} of {} chunks with {} flights of another VM\n", + "PB backing: ctx {} maps {} chunks with {} flights of another VM\n", vm.slot, - mapped, self.pools.chunks.len(), - self.engine - .flights - .iter() - .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) - .count() + others ); return Err(super::boot::eproto()); } - while let Some(chunk) = self.pools.chunks.get(mapped) { - let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; - aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; - vm.vm.append_driver_mappings(aliases)?; - mapped += 1; - vm.pb_chunks.store(mapped, Ordering::Relaxed); - } - Ok(()) + self.pools.map_heap(vm) } - /// The VM whose render work runs: every flight's (`Engine::publish` puts a flight behind - /// another only for the same VM), and it must be the asking context's. EPROTO otherwise. - fn running_vm(&self, ctx: u32) -> Result> { - let flights = &self.engine.flights; - let vm = flights - .first() - .ok_or_else(super::boot::eproto)? - .job - .vm - .clone(); - if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) { - dev_err!( - self.dev(), - "PB growth: ctx {} is not the one VM in flight (ctx {}, {} flights)\n", - ctx, - vm.slot, - flights.len() - ); - return Err(super::boot::eproto()); + /// Backs chunks until the reserve is whole, mapping each into the VMs of the flights (by + /// the one-VM rule, the VM that runs) before the ledger may post from it; any other VM + /// maps it at its next publication ([`Self::map_pb`]). A failure is logged and leaves the + /// reserve short: requests then wait, and the next pass tries again. + pub(crate) fn refill_backing(&mut self) { + // A pending re-initialization is about to drop the posted count: nothing to back for, + // unless a render published before it still runs (the flights behind a fragment kill + // keep asking for growth until they retire). + let render = (0..self.engine.flights.len()).any(|i| self.receipts(i).renders != 0); + if self.recovery.reinit && !render { + return; } - Ok(vm) - } - - /// Backs the PB up to block `end` in the VM that runs, the asking context `ctx`, before an - /// answer posts those blocks (nothing new when they are backed already): chunks of at most - /// [`growth::CHUNK_BLOCKS`] blocks are allocated ([`object::gpu_backing`]), mapped into that - /// VM and kept, one at a time. The VM first catches up ([`Self::map_pb`], which also checks - /// the one-VM rule), so its count equals the chunks. A chunk is kept - /// only once its alias in the VM is held, and the VM's count moves with it; when the alias - /// cannot be held, the alias and the chunk are dropped, neither ever reachable (the leaf is - /// GPU-only: a plain unmap). An error leaves the chunks backed so far for the next request, - /// and nothing is posted. The allocation sleeps (shmem pages with `__GFP_RETRY_MAYFAIL`) - /// while the firmware waits for the answer; how long is not bounded here. - fn back_pb(&mut self, end: u32, ctx: u32) -> Result { - let vm = self.running_vm(ctx)?; - self.map_pb(&vm)?; - while self.pools.backed() < end { - let (first, blocks) = - growth::next_chunk(self.pools.backed(), self.pools.ledger.max).ok_or(ENOSPC)?; - if self.pools.chunks.len() == self.pools.chunks.capacity() { - return Err(ENOSPC); + while self.pools.ledger.refill_due() { + let result = (|| -> Result { + let gem = self.pools.back_chunk()?; + let i = self.pools.chunks.len() - 1; + for flight in self.engine.flights.iter() { + flight.job.vm.map_chunk(i, gem.clone())?; + } + Ok(()) + })(); + match result { + Ok(()) => { + self.pools.ledger.back_chunk(); + self.pools.stats.refills += 1; + dev_dbg!( + self.dev(), + "PB heap: chunk {} backed ({} of {} blocks posted)\n", + self.pools.chunks.len() - 1, + self.pools.ledger.posted, + self.pools.ledger.backed + ); + } + Err(e) => { + // A chunk that could not be mapped into a flight's VM is not counted: drop + // it (and its aliases in the VMs) again. + let backed = (self.pools.ledger.backed / growth::CHUNK_BLOCKS) as usize; + if self.pools.chunks.len() > backed { + for flight in self.engine.flights.iter() { + flight.job.vm.unmap_heap_above(backed); + } + self.pools.chunks.truncate(backed); + } + dev_warn!( + self.dev(), + "PB heap: backing chunk {} failed: {} ({} of {} blocks posted)\n", + backed, + e.to_errno(), + self.pools.ledger.posted, + self.pools.ledger.backed + ); + return; + } } - let chunk = PbChunk::new(&self.pools.drm, first, blocks)?; - let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; - aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; - vm.vm.append_driver_mappings(aliases)?; - // Within the capacity checked above. - self.pools - .chunks - .push_within_capacity(chunk) - .map_err(|_| EIO)?; - vm.pb_chunks - .store(self.pools.chunks.len(), Ordering::Relaxed); } - Ok(()) } /// (Re)initializes the render UMA managers, PB lists and PB manager. `rebuild`: the priming @@ -685,9 +686,12 @@ impl Firmware { /// DEVCTL 7 reports success; at the ledger's maximum the reply is the refusal (+4 = 0), /// which gives the firmware no new pages, and after two refusals requests are left /// unanswered. The firmware imports the blocks itself: P is never rewritten. Only the - /// requests queued before the pass are taken. A failed reply is logged; a request that - /// cannot be processed at all (a firmware object access failed) is logged and left - /// unanswered like an invalid one. Never fails: the poll pass continues. + /// requests queued before the pass are taken. A request with no backed block left waits: + /// it goes back to the head of the queue for the next pass, and this pass ends its answers + /// there. After the answers the reserve is refilled ([`Self::refill_backing`]). A failed + /// reply is logged; a request that cannot be processed at all (a firmware object access + /// failed) is logged and left unanswered like an invalid one. Never fails: the poll pass + /// continues. pub(crate) fn answer_growth(&mut self) -> Result { // Only the requests queued before this pass: a reply's wait drains the rings and may // queue the next request, which the next pass answers (the device lock is released in @@ -697,23 +701,36 @@ impl Firmware { let Ok(request) = self.events.grow.remove(0) else { break; }; - self.pools.stats.requests += 1; - if let Err(e) = self.answer_request(&request) { - self.pools.stats.bad += 1; - dev_err!( - self.dev(), - "PB growth request {} ctx {} not answered: {}\n", - request[buffer::grow_request::COUNTER], - request[buffer::grow_request::CONTEXT], - e.to_errno() - ); + match self.answer_request(&request) { + Ok(true) => self.pools.stats.requests += 1, + Ok(false) => { + // Waits for backing: back at the head, counted once answered. + self.pools.stats.waits += 1; + // The queue has room: this request was just taken from it. + if self.events.grow.insert_within_capacity(0, request).is_err() { + self.pools.stats.bad += 1; + } + break; + } + Err(e) => { + self.pools.stats.requests += 1; + self.pools.stats.bad += 1; + dev_err!( + self.dev(), + "PB growth request {} ctx {} not answered: {}\n", + request[buffer::grow_request::COUNTER], + request[buffer::grow_request::CONTEXT], + e.to_errno() + ); + } } } + self.refill_backing(); Ok(()) } - /// Processes one growth request. - fn answer_request(&mut self, request: &[u32; 14]) -> Result { + /// Processes one growth request: `Ok(false)` when it waits for backing. + fn answer_request(&mut self, request: &[u32; 14]) -> Result { let ctl = self.init.control.clone().ok_or(ENODEV)?; let epoch = ctl.r32(control::HALT_COUNT)? as u64 | (ctl.r32(control::HALT_COUNT + 4)? as u64) << 32; @@ -739,7 +756,7 @@ impl Firmware { self.pools.ledger.max ); } - return Ok(()); + return Ok(true); } // A request from before a halt has no authority over the PB now. if !self @@ -760,20 +777,14 @@ impl Firmware { live as u32, tuning::PB_GROW as u32 ); - return Ok(()); + return Ok(true); } - // Every block the answer posts is backed in the VM that runs before anything of the - // answer is written; the one-VM rule is checked for every answer. If that fails, nothing - // is posted or answered (answer_growth logs it; the deadline applies), and the backing - // made so far stays for the next request. - let end = self.pools.ledger.posted + self.pools.ledger.next_count(); - self.back_pb(end, ctx)?; - self.pools.ledger.backed = self.pools.backed(); + // A grant posts only backed blocks, and every backed chunk is mapped in the VM of the + // flights (at its publication, or at the refill that backed it). let (first, n) = match self.pools.ledger.accept(request) { - growth::Answer::Grant { first, count } => (first, count), - growth::Answer::Refuse => (self.pools.ledger.posted, 0), - // Backed above: no request waits here. - growth::Answer::Wait => return Err(ENOSPC), + Answer::Grant { first, count } => (first, count), + Answer::Refuse => (self.pools.ledger.posted, 0), + Answer::Wait => return Ok(false), }; let heap = client::PB_HEAP; for b in first..first + n { @@ -839,6 +850,6 @@ impl Firmware { ret ); } - Ok(()) + Ok(true) } } diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 0ce3696ebac55..3ed143ee546ad 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -12,11 +12,13 @@ //! its recovery path then apply, without a guarantee that they end the stall. A request from //! before a halt (epoch) or not matching a published render prefix is left unanswered. //! -//! The heap's VA range covers [`MAX_BLOCKS`] blocks; its RAM comes in chunks from the heap -//! base, which the ledger counts as `backed`. A request is granted from the backed blocks above -//! `posted`; a request that finds no backed block waits ([`Answer::Wait`]) and keeps its -//! counter. A re-initialization ([`Ledger::reinit`]) returns the posted count to the initial -//! size and keeps the backing. +//! The heap's VA range covers [`MAX_BLOCKS`] blocks; its RAM comes in chunks of +//! [`CHUNK_BLOCKS`] blocks from the heap base, which the ledger counts as `backed`. A request is +//! granted from the backed blocks above `posted`. A chunk is due ([`Ledger::refill_due`]) +//! whenever fewer than [`RESERVE_BLOCKS`] backed blocks remain above the posted count, so that +//! it can be backed after a reply; a request that finds no backed block waits +//! ([`Answer::Wait`]) and keeps its counter. A re-initialization ([`Ledger::reinit`]) returns +//! the posted count to the initial size and keeps the backing. //! //! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. @@ -52,6 +54,13 @@ pub(crate) const MAX_REFUSALS: u32 = 2; /// Blocks of each PB backing chunk (16 MiB). pub(crate) const CHUNK_BLOCKS: u32 = 128; +/// Backed blocks kept above the posted count: at 1-2 ms per request of ten blocks, 13 requests +/// of lead for the allocation of the next chunk. +pub(crate) const RESERVE_BLOCKS: u32 = CHUNK_BLOCKS; + +/// Chunks of the whole heap. +pub(crate) const MAX_CHUNKS: u32 = MAX_BLOCKS / CHUNK_BLOCKS; + /// The answer to an admitted growth request. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum Answer { @@ -64,24 +73,6 @@ pub(crate) enum Answer { Wait, } -/// The backing chunk that follows `backed` blocks of a ledger with maximum `max`: its first block -/// and size, at most [`CHUNK_BLOCKS`] and never past `max`; none from `max` on. -pub(crate) const fn next_chunk(backed: u32, max: u32) -> Option<(u32, u32)> { - if backed >= max { - None - } else if max - backed < CHUNK_BLOCKS { - Some((backed, max - backed)) - } else { - Some((backed, CHUNK_BLOCKS)) - } -} - -/// The chunks that back a ledger with `initial` and `max` blocks at most: the initial one and -/// [`next_chunk`]'s from `initial` to `max`. -pub(crate) const fn max_chunks(initial: u32, max: u32) -> usize { - 1 + max.saturating_sub(initial).div_ceil(CHUNK_BLOCKS) as usize -} - /// The most blocks a ledger posts: the mapped heap, and below K, so that the block_ctl+4 /// producer (posted modulo K) is never 0, the value of an empty block list. pub(crate) const MAX_POSTED: u32 = if MAX_BLOCKS < BLOCK_LIST_RECORDS { @@ -125,11 +116,32 @@ impl Ledger { self.posted % BLOCK_LIST_RECORDS } + /// Records one more backed chunk. + pub(crate) const fn back_chunk(&mut self) { + self.backed += CHUNK_BLOCKS; + } + /// Backed blocks above the posted count. pub(crate) const fn spare(&self) -> u32 { self.backed.saturating_sub(self.posted) } + /// Whether another chunk is due: fewer than the reserve above the posted count and the heap + /// not yet backed whole. + pub(crate) const fn refill_due(&self) -> bool { + self.backed < MAX_BLOCKS && self.spare() < RESERVE_BLOCKS + } + + /// Chunks that `blocks` posted blocks and the reserve need. + pub(crate) const fn chunks_for(blocks: u32) -> u32 { + let n = (blocks + RESERVE_BLOCKS).div_ceil(CHUNK_BLOCKS); + if n > MAX_CHUNKS { + MAX_CHUNKS + } else { + n + } + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -148,12 +160,6 @@ impl Ledger { && request_epoch(request) == epoch } - /// The blocks the next accepted request posts with the backing it needs: at most - /// [`GROW_STEP`], none at the maximum. - pub(crate) fn next_count(&self) -> u32 { - GROW_STEP.min(self.max - self.posted) - } - /// Accepts an admitted request. Granted (at most [`GROW_STEP`] blocks, and no more than are /// backed) or refused at the maximum, the next expected counter follows it. A request that /// finds no backed block waits: its counter is not consumed, and it is accepted again once a @@ -387,6 +393,105 @@ mod tests { assert_eq!(l.post(10), 10); } + #[test] + fn backing() { + // 64 initial blocks: two chunks (the initial posting and the reserve). + assert_eq!(Ledger::chunks_for(64), 2); + assert_eq!(Ledger::chunks_for(1024), 9); + assert_eq!(Ledger::chunks_for(4095), MAX_CHUNKS); + assert_eq!(MAX_CHUNKS, 32); + let mut l = Ledger::new(true, 64, 4095); + assert_eq!(l.spare(), 0); + assert!(l.refill_due()); + // Nothing backed: a request waits and the counter does not advance. + let q = request(3, 0, 1); + assert_eq!(l.accept(&q), Answer::Wait); + assert_eq!(l.next_request, 0); + l.back_chunk(); + l.back_chunk(); + assert_eq!((l.backed, l.spare()), (256, 192)); + assert!(!l.refill_due()); + // Granting until the reserve line, then a refill is due; at the end of the backing the + // grant is clipped and the next request waits. + assert_eq!( + l.accept(&q), + Answer::Grant { + first: 64, + count: 10 + } + ); + assert_eq!(l.post(10), 74); + assert_eq!(l.next_request, 1); + l.posted = 128; + assert!(!l.refill_due()); + l.posted = 129; + assert!(l.refill_due()); + l.posted = 250; + assert_eq!( + l.accept(&request(3, 1, 1)), + Answer::Grant { + first: 250, + count: 6 + } + ); + l.post(6); + assert_eq!(l.accept(&request(3, 2, 1)), Answer::Wait); + // The heap end: no refill past the whole heap, the maximum refuses. + l.backed = MAX_BLOCKS; + l.posted = 4090; + assert!(!l.refill_due()); + assert_eq!( + l.accept(&request(3, 2, 1)), + Answer::Grant { + first: 4090, + count: 5 + } + ); + l.post(5); + assert_eq!(l.accept(&request(3, 3, 1)), Answer::Refuse); + // Re-initialization keeps the backing. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 4096)); + } + + /// Every ledger the constructor makes, the driver's included, backed a chunk at a time + /// whenever the reserve is short: no grant posts a block past the backing, no request waits, + /// and the backing ends within the heap. + #[test] + fn backing_ahead_of_every_grant() { + let values = [ + 0, 1, MIN_BLOCKS, 65, 1023, 1024, 1025, 3072, 4094, MAX_POSTED, 5000, + ]; + for grow in [true, false] { + for blocks in values { + for max_blocks in values { + let mut l = Ledger::new(grow, blocks, max_blocks); + for _ in 0..Ledger::chunks_for(l.initial) { + l.back_chunk(); + } + let mut counter = 0; + loop { + match l.accept(&request(5, counter, 1)) { + Answer::Grant { first, count } => { + assert_eq!(first, l.posted); + assert!(count >= 1 && first + count <= l.backed); + l.post(count); + } + Answer::Refuse => break, + Answer::Wait => panic!("waited at {} of {}", l.posted, l.backed), + } + counter += 1; + while l.refill_due() { + l.back_chunk(); + } + } + assert_eq!(l.posted, l.max); + assert!(l.max <= l.backed && l.backed <= MAX_BLOCKS); + } + } + } + } + #[test] fn grants_only_backed_blocks() { let mut l = Ledger::new(true, 64, 4095); diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index ccb4f77921e0c..884111d282bdd 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -10,8 +10,9 @@ //! ([`object::Objects::clients`]) as driver mappings at their fixed GVAs, and reserve //! [`super::hw::client::CLIENT_APERTURE`], the pass-slot range and the PB page list //! (`vm.install_driver_mappings`): user binds there fail (EINVAL). The aperture holds the -//! whole PB range, whose backing is mapped later, before the VM's first prefix is published -//! (`Firmware::map_pb`, counted in [`VmState::pb_chunks`]); +//! whole PB range, whose backing chunks are mapped later, before the VM's prefixes are +//! published (`Firmware::map_pb`) and, for the VM of the flights, when a chunk is backed +//! ([`VmState::map_heap`], [`VmState::map_chunk`]); //! - bind it to an exclusive TTBAT slot ([`mmu::Uat::bind`]) for as long as its [`VmState`] //! lives; at most [`super::hw::sizes::MAX_VMS`] such VMs (ENOSPC beyond, checked before //! binding so the slot allocator never blocks). @@ -29,7 +30,7 @@ //! alike. use core::ops::Range; -use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicU32, Ordering}; use kernel::{ drm::gem::BaseObject, new_mutex, @@ -104,6 +105,20 @@ impl ClientAlias { } } +/// A PB heap chunk mapped in a VM at its fixed GVA ([`super::pools::chunk_gva`]). +struct HeapMapping { + gem: ARef, + _mapping: mmu::KernelMapping, +} + +/// The PB heap chunks mapped in a VM, in heap order (chunk `i` at `chunk_gva(i)`). Changed +/// under the device lock (refill, publication). +#[pin_data] +struct HeapMappings { + #[pin] + list: Mutex>, +} + /// G15 state of one user VM, shared by the registry, its queues and its jobs. pub(crate) struct VmState { /// The `mmu::Vm` identifier (`Gpu::ids.vm`). @@ -117,9 +132,8 @@ pub(crate) struct VmState { /// The exclusive slot binding, released before the live count drops. bind: Option, registry: Arc, - /// PB backing chunks mapped into the VM, from the first (`Firmware::map_pb`); changed only - /// under the device lock. - pub(crate) pb_chunks: AtomicUsize, + /// The PB heap chunks mapped in this VM. + heap: Arc, } impl VmState { @@ -127,6 +141,62 @@ impl VmState { pub(crate) fn prune_registry(&self) { self.registry.prune(); } + + /// Maps PB heap chunk `i`, the next one this VM lacks, at its GVA. The alias is held only + /// once it is in the list. + fn map_one(&self, heap: &mut KVec, i: usize, gem: ARef) -> Result { + if i != heap.len() { + return Err(EINVAL); + } + heap.reserve(1, GFP_KERNEL)?; + let mapping = self.vm.map_at( + super::pools::chunk_gva(i), + super::pools::CHUNK_BYTES, + &gem, + mmu::Prot::from_pte(hw::pte::LEAF_CACHED), + false, + )?; + heap.push_within_capacity(HeapMapping { + gem, + _mapping: mapping, + }) + .map_err(|_| ENOMEM) + } + + /// Maps the backed chunks (`gems`, in heap order) this VM does not hold yet. A chunk whose + /// object changed since (freed and backed again) is unmapped first, with everything above + /// it, and so are chunks past the backing. + pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result { + let mut heap = self.heap.list.lock(); + let mut n = 0; + for (i, gem) in gems.enumerate() { + n = i + 1; + if let Some(have) = heap.get(i) { + if core::ptr::eq(&*have.gem, &*gem) { + continue; + } + heap.truncate(i); + } + self.map_one(&mut heap, i, gem)?; + } + heap.truncate(n); + Ok(()) + } + + /// Maps the newly backed chunk `i` into this VM (the VM of a flight, at a refill). + pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result { + let mut heap = self.heap.list.lock(); + if i < heap.len() { + // Already mapped (a flight of the same VM came first). + return Ok(()); + } + self.map_one(&mut heap, i, gem) + } + + /// Unmaps the heap chunks from index `keep` on. + pub(crate) fn unmap_heap_above(&self, keep: usize) { + self.heap.list.lock().truncate(keep); + } } impl Drop for VmState { @@ -292,6 +362,13 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { let status = vm.status().ok_or(EINVAL)?.clone(); let (mappings, reserved) = Registry::driver_mappings(&vm, aliases)?; vm.install_driver_mappings(mappings, reserved)?; + // No PB chunk yet: they are mapped before the VM's first prefix (`Firmware::map_pb`). + let heap = Arc::pin_init( + pin_init!(HeapMappings { + list <- new_mutex!(KVec::new(), "g15::HeapMappings::list"), + }), + GFP_KERNEL, + )?; // The context names the root only now: everything above is already visible to the walker. let bind = gpu.uat.bind(&vm)?; let slot = bind.slot(); @@ -305,7 +382,7 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { vm: vm.clone(), bind: Some(bind), registry: registry.clone(), - pb_chunks: AtomicUsize::new(0), + heap, }, GFP_KERNEL, )?; diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 67fa40aab75f3..c1e1f8998402d 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -3,16 +3,17 @@ """Run the G15 PB growth answer and its on-demand backing, actual bodies, against a model. The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), -answer_request(), map_pb(), running_vm() and back_pb(), Pools::soft_max() and backed(), -PbChunk and Pools::new's PB backing statements; pb_max_blocks() from params.rs; the PB sizes, -tuning and PB leaf word from hw.rs; Prot::from_pte() and is_cached_noncoherent() from -pgtable.rs; and the parameter's default from asahi_core.rs. It includes pools/growth.rs, -fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, -the GEM backing allocation, the shared layer's map_at() and append_driver_mappings(), and the -DEVCTL wait, which logs every allocation, alias map, append, unmap, free, block record, -barrier, producer store, reply and log line in order. Allocation, alias map and append -failures can be injected. Source checks: publish_prefix_into() starts with the PB catch-up, -and pools.rs asserts the PB leaf GPU-only at compile time. +answer_request(), map_pb() and refill_backing(), Pools::soft_max(), map_heap() and +back_chunk(), PbChunk, chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from +vm.rs VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() from +params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and +is_cached_noncoherent() from pgtable.rs; and the parameter's default from asahi_core.rs. It +includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model +of the firmware objects, the GEM backing allocation, the shared layer's map_at() and the DEVCTL +wait, which logs every allocation, alias map, unmap, free, block record, barrier, producer +store, reply and log line in order. Allocation and alias map failures can be injected. Source +checks: publish_prefix_into() starts with the PB catch-up, and pools.rs asserts the PB leaf +GPU-only at compile time. What it shows: - a poll pass answers only the requests queued when it starts, in order; one the reply's wait @@ -21,22 +22,24 @@ offset, each producer store follows its records and a barrier, each reply follows its producer store and a barrier, and every producer is in 1..=4095, ending at 4095; - at the maximum two refusals are answered; further requests get no reply, count as bad and - warn once until the ledger is made again, as init() does; + warn once until the ledger is reset, as init() does; - the soft maximum is the parameter clamped to 64..=4095 and then to the ledger's initial..=max; growth answers above it are logged at info level, and the one that reaches it exactly is not; - invalid requests (type, slot, stale counter, epoch, no live render) get no reply, and requests rejected so allocate nothing; -- the backing: the initial blocks only when the GPU starts; chunks of at most 16 MiB, allocated - and held by the running VM before any record or producer names their blocks, never past the - maximum, at most growth::max_chunks(); a failed allocation, alias map or append posts and - answers nothing, leaves the request counter and the VM's count alone and drops what is not - held, and the next request is answered; a failed reply keeps everything; -- a VM's count always equals the aliases it holds, contiguous from the heap's start; a VM that - runs after another grew the backing maps it first (at publication, and again before it grows); - a partial catch-up counts only held aliases; -- the one-VM rule: a catch-up with a flight of another VM, or a count past the chunks, and a - grant with flights of two VMs, are refused (EPROTO) before anything is mapped or posted; +- the backing: the chunks of the initial posting and one reserve chunk when the GPU starts; + chunks of 16 MiB, backed after the replies of a pass (never between them) while fewer than a + reserve chunk remain, mapped into the VM of the flights before any record or producer names + their blocks, never past the heap; with the reserve dry a request waits at the head of the + queue, keeps its counter and is answered once a chunk is backed; a chunk that cannot be mapped + into the running VM is dropped again; a pending re-initialization backs nothing while no + render is published, and still backs for a render published before it; a failed reply keeps + everything; +- a VM's aliases are the chunks, contiguous from the heap's start; a VM that runs after another + grew the backing maps it first (at publication); a partial catch-up keeps the aliases made; +- the one-VM rule: a catch-up with a flight of another VM is refused (EPROTO) before anything + is mapped; - the ledger reset of Pools::init() re-posts from the backing without allocating; - the PB leaf decodes as GPU-only, not cached-noncoherent: dropping an alias is a plain unmap; - Ledger::new() is Ledger::new() as of dba2a110de00 (the carried upstream constructor), capped at @@ -47,8 +50,7 @@ refusal or an unanswered request. The model's VMs have no open/closed state: that a closed VM whose jobs still run keeps its aliases comes from the VM lifetime source (Job and VmJobGuard hold the VM), not from these tests. The ledger reset is not a recovery. The shared layer's -mapping and its rollback are modelled, not run. Running only running_vm()'s or only map_pb()'s -one-VM check on a grant is not distinguishable: each covers it. +mapping is modelled, not run. With --mutations, each mutation below must fail its intended test with the expected message, and the control (a driver maximum of the whole 4096-block heap) must pass, held by the ledger's cap. @@ -75,6 +77,7 @@ HW = f"{G15}/hw.rs" CORE = "drivers/gpu/drm/asahi/asahi_core.rs" ENGINE = f"{G15}/engine.rs" +VM = f"{G15}/vm.rs" PGTABLE = "drivers/gpu/drm/asahi/pgtable.rs" OLD_REF = "dba2a110de00743eeecd41edf23e54ede4491d5e" @@ -97,12 +100,13 @@ } }''' -# Pools::new's PB backing, run verbatim by the model. -POOLS_NEW_BACKING = (' // Backing for the initial blocks; growth adds the rest.\n', ' ledger.backed = ') +# Pools::new's PB backing, run verbatim by the model (up to its log line). +POOLS_NEW_BACKING = (' // Backing for the initial blocks and the reserve; growth adds the rest.\n', + ' dev_info!(\n') # The compile-time proof that a PB alias drop is a plain unmap. PB_LEAF_ASSERT = 'static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent());' -# The two statements of Pools::init() that start a new ledger. +# The two statements of Pools::init() that reset the ledger. LEDGER_RESET = ['self.ledger.reinit();', 'self.exhausted_logged = false;'] @@ -112,9 +116,9 @@ # Leave the last request queued before the pass for later. 'skip-last-queued': (POOLS, 'for _ in 0..queued {', 'for _ in 0..queued.saturating_sub(1) {'), # End the pass at a request that cannot be processed. - 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', - ' self.pools.stats.bad += 1;\n return Ok(());\n' - ' dev_err!('), + 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', + ' self.pools.stats.bad += 1;\n return Ok(());\n' + ' dev_err!('), # Answer three refusals at the maximum. 'three-refusals': (GROWTH, 'pub(crate) const MAX_REFUSALS: u32 = 2;', 'pub(crate) const MAX_REFUSALS: u32 = 3;'), # Answer refusals for ever. @@ -140,42 +144,33 @@ 'let nb = clamp(blocks, MIN_BLOCKS, MAX_BLOCKS);\n' ' (nb, clamp(max_blocks, nb, MAX_BLOCKS))\n' ' } else {\n (MAX_BLOCKS, MAX_BLOCKS)'), - # Post blocks without backing them first (the ledger told the whole heap is backed). - 'grant-without-backing': (POOLS, ' self.back_pb(end, ctx)?;\n self.pools.ledger.backed = self.pools.backed();\n', - ' self.pools.ledger.backed = buffer::MAX_BLOCKS;\n'), - # Back the blocks only after the request is accepted. - 'back-after-accept': [(POOLS, ' self.back_pb(end, ctx)?;\n self.pools.ledger.backed = self.pools.backed();\n', - ' self.pools.ledger.backed = buffer::MAX_BLOCKS;\n'), - (POOLS, ' growth::Answer::Wait => return Err(ENOSPC),\n };\n', - ' growth::Answer::Wait => return Err(ENOSPC),\n };\n self.back_pb(end, ctx)?;\n')], - # Count a catch-up alias before the VM holds it. - 'catch-up-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n mapped += 1;\n' - ' vm.pb_chunks.store(mapped, Ordering::Relaxed);', - ' mapped += 1;\n vm.pb_chunks.store(mapped, Ordering::Relaxed);\n' - ' vm.vm.append_driver_mappings(aliases)?;'), - # Count a growth chunk's alias before the VM holds it. - 'growth-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.', - ' vm.pb_chunks.store(self.pools.chunks.len() + 1, Ordering::Relaxed);\n' - ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.'), - # Keep a chunk whose alias the VM never held. - 'keep-unappended-chunk': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.\n' - ' self.pools\n .chunks\n .push_within_capacity(chunk)\n' - ' .map_err(|_| EIO)?;', - ' // Within the capacity checked above.\n self.pools\n .chunks\n' - ' .push_within_capacity(chunk)\n .map_err(|_| EIO)?;\n' - ' vm.vm.append_driver_mappings(aliases)?;'), - # Grow without catching the running VM up first. - 'no-growth-catch-up': (POOLS, ' let vm = self.running_vm(ctx)?;\n self.map_pb(&vm)?;\n', - ' let vm = self.running_vm(ctx)?;\n'), + # Grant without the backing: neither wait nor clip. + 'grant-without-backing': [(GROWTH, ' if room != 0 && self.spare() == 0 {\n return Answer::Wait;\n }\n', ''), + (GROWTH, 'count: GROW_STEP.min(room).min(self.spare()),', 'count: GROW_STEP.min(room),')], + # Allocate before the pass's answers. + 'refill-before-answers': [(POOLS, ' self.refill_backing();\n Ok(())', ' Ok(())'), + (POOLS, ' let queued = self.events.grow.len();\n', + ' self.refill_backing();\n let queued = self.events.grow.len();\n')], + # Back a chunk without mapping it into the running VM. + 'refill-maps-no-flight': (POOLS, ' flight.job.vm.map_chunk(i, gem.clone())?;\n', + ' let _ = (flight, i, gem.clone());\n'), + # Drop a request that waits for backing. + 'no-wait-requeue': (POOLS, ' if self.events.grow.insert_within_capacity(0, request).is_err() {\n' + ' self.pools.stats.bad += 1;\n }\n', ''), + # Keep a chunk that the running VM could not map. + 'refill-failure-keeps-chunk': (POOLS, ' self.pools.chunks.truncate(backed);\n', ''), + # Back chunks while a re-initialization is pending. + 'refill-during-reinit': (POOLS, ' if self.recovery.reinit && !render {\n return;\n }\n', ''), + # Skip the refill on any pending re-initialization, also with a live render (a stall). + 'refill-skip-with-live-render': (POOLS, 'if self.recovery.reinit && !render {', 'if self.recovery.reinit {'), + # Refill for any live flight, also compute-only ones that cannot ask for growth. + 'refill-for-any-flight': (POOLS, 'if self.recovery.reinit && !render {', + 'if self.recovery.reinit && self.engine.flights.is_empty() {'), # Catch up without the one-VM check. - 'no-activation-guard': (POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), - # Neither the catch-up's nor the grant's one-VM check (each alone covers a grant). - 'no-same-vm-checks': [(POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), - (POOLS, 'if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) {', - 'if vm.slot != ctx {')], + 'no-activation-guard': (POOLS, '.filter(|f| !core::ptr::eq(&*f.job.vm, vm))', '.filter(|_| false)'), # Back the whole heap when the GPU starts. - 'initial-backs-whole-heap': (POOLS, 'chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?;', - 'chunks.push(PbChunk::new(dev, 0, buffer::MAX_BLOCKS)?, GFP_KERNEL)?;'), + 'initial-backs-whole-heap': (POOLS, 'for _ in 0..Ledger::chunks_for(ledger.initial) {', + 'for _ in 0..growth::MAX_CHUNKS {'), # Growth chunks of 32 MiB. 'chunks-above-16mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 128;', 'pub(crate) const CHUNK_BLOCKS: u32 = 256;'), # A firmware-visible cached PB leaf (its drop would ask the firmware to flush). @@ -185,7 +180,7 @@ # The test each mutation must fail, and a piece of its failure message. INTENDED = { - 'drain-until-empty': ('queued_snapshot_keeps_order', 'unbounded answering in one poll pass'), + 'drain-until-empty': ('queued_snapshot_keeps_order', 'replies in the first pass'), 'skip-last-queued': ('queued_snapshot_keeps_order', 'replies in the first pass'), 'return-after-request-error': ('unprocessed_request_does_not_end_the_pass', 'requests taken in the pass'), 'three-refusals': ('default_growth_to_exhaustion', 'want Some(Log("warn"'), @@ -197,14 +192,15 @@ 'soft-boundary-inclusive': ('soft_max_boundaries', 'got Some(Log("info", "")), want Some(Log("dbg", ""))'), 'soft-max-unclamped': ('soft_max_bounds', 'soft_max, param 0'), 'uncapped-ledger': ('constructor_is_the_old_one_capped', 'new(true, '), - 'grant-without-backing': ('growth_backs_before_posting', 'record of block'), - 'back-after-accept': ('allocation_failure_posts_nothing', 'ledger after a backing failure'), - 'catch-up-cursor-before-append': ('partial_catch_up_counts_only_held_aliases', 'count after a failed append'), - 'growth-cursor-before-append': ('append_failure_posts_nothing', 'count differs from held aliases'), - 'keep-unappended-chunk': ('append_failure_posts_nothing', 'chunks after a backing failure'), - 'no-growth-catch-up': ('growth_catches_the_running_vm_up_first', 'running VM not caught up'), + 'grant-without-backing': ('dry_reserve_waits_and_keeps_order', 'record of block'), + 'refill-before-answers': ('allocation_follows_the_replies', 'allocation before a reply'), + 'refill-maps-no-flight': ('growth_backs_before_posting', 'record of block'), + 'no-wait-requeue': ('dry_reserve_waits_and_keeps_order', 'waiting request lost'), + 'refill-failure-keeps-chunk': ('refill_map_failure_drops_the_chunk', 'chunk kept after a failed map'), + 'refill-during-reinit': ('pending_reinit_without_flights_backs_nothing', 'backed during a pending re-initialization'), + 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), + 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), - 'no-same-vm-checks': ('grant_refuses_flights_of_two_vms', 'grant with flights of two VMs'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), 'pb-leaf-fw-visible': ('pb_leaf_is_gpu_only', 'PB leaf not GPU-only'), @@ -294,6 +290,26 @@ def consts(text, names): pub fn last(&self) -> Option<&T> { self.0.last() } + pub fn truncate(&mut self, n: usize) { + self.0.truncate(n) + } + pub fn reserve(&mut self, n: usize, _: Flags) -> Result { + self.0.reserve(n); + Ok(()) + } + pub fn iter(&self) -> std::slice::Iter<'_, T> { + self.0.iter() + } +} +/// A kernel mutex, single-threaded here. +pub struct Mutex(pub RefCell); +impl Mutex { + pub fn new(value: T) -> Self { + Mutex(RefCell::new(value)) + } + pub fn lock(&self) -> std::cell::RefMut<'_, T> { + self.0.borrow_mut() + } } #[macro_export] macro_rules! static_assert { @@ -313,20 +329,17 @@ def consts(text, names): /// Backing object `id` of this many bytes allocated, or freed (its last reference gone). Alloc(u32, usize), Free(u32), - /// A PB alias of object `id` at GVA in the VM with this context, made, held by the VM - /// (appended to its driver mappings), or unmapped. + /// A PB alias of object `id` at GVA in the VM with this context, made or unmapped. Map(u32, u64, usize, u32), - Append(u32, u64), Unmap(u32, u64), } thread_local! { pub static LOG: RefCell> = RefCell::new(Vec::new()); pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); - /// Failures to inject: the next this many allocations, alias maps, appends fail. + /// Failures to inject: the next this many allocations or alias maps fail. pub static FAIL_ALLOC: Cell = Cell::new(0); pub static FAIL_MAP: Cell = Cell::new(0); pub static MAP_OK_BEFORE_FAIL: Cell = Cell::new(0); - pub static FAIL_APPEND: Cell = Cell::new(0); pub static NEXT_ID: Cell = Cell::new(1); } /// Takes one injected failure of `which`. @@ -393,7 +406,7 @@ def consts(text, names): } } -/// The shared layer's VM: kernel aliases at fixed addresses, held by the VM once appended. +/// The shared layer's VM: kernel aliases at fixed addresses. pub mod mmu { use crate::prelude::*; pub struct Prot(pub u64); @@ -416,11 +429,10 @@ def consts(text, names): } pub struct Vm { pub slot: u32, - pub held: std::cell::RefCell>, } impl Vm { pub fn new(slot: u32) -> Self { - Vm { slot, held: Default::default() } + Vm { slot } } pub fn map_at( &self, @@ -441,24 +453,9 @@ def consts(text, names): if !skip && crate::fail(&crate::FAIL_MAP) { return Err(ENOMEM); } - // Leaves already in use: EBUSY (prepare_map). - let end = addr + size as u64; - if self.held.borrow().iter().any(|m| m.gva < end && addr < m.gva + m.size as u64) { - return Err(EBUSY); - } crate::ev(crate::Ev::Map(self.slot, addr, size, gem.id)); Ok(KernelMapping { slot: self.slot, gva: addr, size, prot: prot.0, gem: gem.clone() }) } - pub fn append_driver_mappings(&self, mappings: KVec) -> Result { - if crate::fail(&crate::FAIL_APPEND) { - return Err(ENOMEM); // the mappings drop here - } - for m in mappings.0 { - crate::ev(crate::Ev::Append(self.slot, m.gva)); - self.held.borrow_mut().push(m); - } - Ok(()) - } } } ''' @@ -481,14 +478,26 @@ def consts(text, names): pub struct Init { pub control: Option>, } + /// vm.rs's chunk mapping, its fields as the driver names them. + pub struct HeapMapping { + pub gem: ARef, + pub _mapping: mmu::KernelMapping, + } + pub struct HeapMappings { + pub list: Mutex>, + } pub struct VmState { pub slot: u32, pub vm: mmu::Vm, - pub pb_chunks: AtomicUsize, + pub heap: Arc, } impl VmState { pub fn new(slot: u32) -> Arc { - Arc::new(VmState { slot, vm: mmu::Vm::new(slot), pb_chunks: AtomicUsize::new(0) }) + Arc::new(VmState { + slot, + vm: mmu::Vm::new(slot), + heap: Arc::new(HeapMappings { list: Mutex::new(KVec(Vec::new())) }), + }) } } pub struct Job { @@ -500,10 +509,13 @@ def consts(text, names): pub struct Engine { pub flights: Vec, } + pub struct Recovery { + pub reinit: bool, + } pub struct Receipts { pub renders: u32, } - /// A firmware object: the block list, block_ctl or the heap. + /// A firmware object: the block list or block_ctl. pub struct FwObject { pub gva: Option, pub size: usize, @@ -536,6 +548,8 @@ def consts(text, names): pub grown: u64, pub refused: u64, pub bad: u64, + pub waits: u64, + pub refills: u64, } pub struct Live { pub block_ctl: FwObject, @@ -556,6 +570,10 @@ def consts(text, names): pub fn chunk_capacity(&self) -> usize { self.chunks.capacity() } + /// The backing objects' ids, in heap order. + pub fn chunk_ids(&self) -> Vec { + self.chunks.iter().map(|c| c.backing.gem.id).collect() + } } pub struct Queue(pub std::collections::VecDeque<[u32; 14]>); impl Queue { @@ -569,6 +587,11 @@ def consts(text, names): assert_eq!(i, 0); self.0.pop_front().ok_or(()) } + pub fn insert_within_capacity(&mut self, i: usize, q: [u32; 14]) -> core::result::Result<(), [u32; 14]> { + assert_eq!(i, 0); + self.0.push_front(q); + Ok(()) + } } pub struct Events { pub grow: Queue, @@ -601,6 +624,7 @@ def consts(text, names): pub pools: Pools, pub init: Init, pub engine: Engine, + pub recovery: Recovery, pub renders: Vec, pub fw: FwModel, } @@ -616,7 +640,7 @@ def consts(text, names): crate::take(); f } - /// Pools::new's PB backing: the initial chunk, room for every chunk the ledger needs. + /// Pools::new's PB backing: the initial chunks and the reserve, room for the heap. pub fn without_flights() -> Self { let drm = ARef::new(AsahiDevice); let (ledger, chunks) = pools_new_backing(&drm).unwrap(); @@ -639,6 +663,7 @@ def consts(text, names): }, init: Init { control: Some(std::sync::Arc::new(Ctl { epoch: 0x1_0000_0003 })) }, engine: Engine { flights: Vec::new() }, + recovery: Recovery { reinit: false }, renders: vec![1], fw: FwModel { ctx: 5, @@ -695,10 +720,10 @@ def consts(text, names): use crate::g15::firmware::{Firmware, VmState, Flight, Job, HEAP}; use crate::g15::hw::{client, pte}; use crate::g15::fw::buffer::{BLOCK_BYTES, MAX_BLOCKS}; - use crate::g15::pools::growth::{max_chunks, CHUNK_BLOCKS}; + use crate::g15::pools::growth::{CHUNK_BLOCKS, MAX_CHUNKS}; + use crate::g15::pools::{chunk_gva, CHUNK_BYTES}; use crate::g15::pgtable::Prot; - use crate::{EPROTO, FAIL_ALLOC, FAIL_APPEND, FAIL_MAP}; - use core::sync::atomic::Ordering; + use crate::{EPROTO, FAIL_ALLOC, FAIL_MAP}; use std::sync::Arc; use crate::g15::fw::buffer::{self, grow_reply, BLOCK_LIST_RECORDS, GROW_STEP}; use crate::g15::fw::channels::devctl; @@ -779,7 +804,7 @@ def consts(text, names): fn levels_only(evs: Vec) -> Vec { evs.into_iter() - .filter(|e| !is_backing(e)) + .filter(|e| !is_backing(e) && !matches!(e, Ev::Log(_, m) if m.starts_with("PB heap:"))) .map(|e| match e { Ev::Log(l, _) => Ev::Log(l, String::new()), e => e, @@ -892,6 +917,7 @@ def consts(text, names): f.queue(); f.queue(); take(); + let mapped = f.pools.ledger.backed as u64; f.pass(); let log = take(); let replies: Vec = log @@ -983,7 +1009,7 @@ def consts(text, names): assert_eq!(log.len(), 4); assert_eq!(f.pools.stats.bad, 4); let mut fresh = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); - fresh.backed = fresh.initial; + fresh.backed = Ledger::chunks_for(fresh.initial) * CHUNK_BLOCKS; assert_eq!(f.pools.ledger, fresh); // No render published: not live. f.renders[0] = 0; @@ -1002,54 +1028,43 @@ def consts(text, names): } fn is_backing(e: &Ev) -> bool { - matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Append(..) | Ev::Unmap(..)) + matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Unmap(..)) } - /// A VM's PB aliases: its count, and the aliases it holds, from block 0 in order. - fn assert_cursor(vm: &VmState, chunks: usize) { - let held = vm.vm.held.borrow(); - assert_eq!(vm.pb_chunks.load(Ordering::Relaxed), held.len(), "count differs from held aliases"); - assert!(held.len() <= chunks); - let mut gva = client::PB_HEAP; - for m in held.iter() { - assert_eq!(m.gva, gva, "aliases not contiguous from the heap's start"); - assert_eq!(m.prot, pte::LEAF_CACHED); - gva += m.size as u64; + /// A VM's PB aliases: exactly the first `chunks` chunks, in order, each the whole chunk at + /// its GVA with the PB leaf, each of the chunk's object. + fn assert_cursor(f: &Firmware, vm: &VmState, chunks: usize) { + let heap = vm.heap.list.lock(); + assert_eq!(heap.len(), chunks, "VM aliases differ from the chunks"); + let ids = f.pools.chunk_ids(); + for (i, m) in heap.iter().enumerate() { + assert_eq!(m._mapping.gva, chunk_gva(i), "aliases not contiguous from the heap's start"); + assert_eq!(m._mapping.size, CHUNK_BYTES); + assert_eq!(m._mapping.prot, pte::LEAF_CACHED); + assert_eq!(m.gem.id, ids[i], "alias of another object"); } } /// In the order of `log`: no block record and no producer past the blocks the VM with - /// context `slot` holds aliases for at that point. + /// context `slot` has mapped at that point (`start` blocks before the log). fn assert_posted_backed(log: &[Ev], slot: u32, start: u64) { let mut mapped = start; for e in log { match e { - Ev::Append(s, gva) if *s == slot => { - assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "append out of order"); + Ev::Map(s, gva, size, _) if *s == slot => { + assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "map out of order"); + mapped += *size as u64 / BLOCK_BYTES; } - Ev::Map(s, gva, size, _) if *s == slot && *gva == client::PB_HEAP + mapped * BLOCK_BYTES => { - // Counted at its append. - let _ = size; + Ev::Unmap(s, gva) if *s == slot => { + mapped = mapped.min((gva - client::PB_HEAP) / BLOCK_BYTES); } - _ => {} - } - if let Ev::Append(s, gva) = e { - if *s == slot { - let size = log - .iter() - .find_map(|m| match m { - Ev::Map(ms, mg, size, _) if ms == s && mg == gva => Some(*size as u64), - _ => None, - }) - .unwrap(); - mapped += size / BLOCK_BYTES; + Ev::Record(off, _) => { + assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); } - } - if let Ev::Record(off, _) = e { - assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); - } - if let Ev::Producer(p) = e { - assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + Ev::Producer(p) => { + assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + } + _ => {} } } } @@ -1075,17 +1090,20 @@ def consts(text, names): let mut f = Firmware::without_flights(); let log = take(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); - assert_eq!(log, vec![Ev::Alloc(1, initial as usize * BLOCK_BYTES as usize)], "startup allocation"); - assert_eq!((f.pools.backed(), f.pools.chunk_count()), (initial, 1)); - assert_eq!(f.pools.chunk_capacity(), max_chunks(initial, max)); - assert_eq!(max_chunks(initial, max), 25); + let chunks = Ledger::chunks_for(initial) as usize; + assert_eq!(chunks, 9); + let want: Vec = (1..=chunks as u32).map(|id| Ev::Alloc(id, CHUNK_BYTES)).collect(); + assert_eq!(log, want, "startup allocation"); + assert_eq!((f.pools.ledger.backed, f.pools.chunk_count()), (chunks as u32 * CHUNK_BLOCKS, chunks)); + assert_eq!(f.pools.chunk_capacity(), MAX_CHUNKS as usize); + assert!(max <= MAX_CHUNKS * CHUNK_BLOCKS); // The first prefix of a VM maps the backing, nothing else. let vm = VmState::new(7); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); f.map_pb(&vm).unwrap(); - let size = initial as usize * BLOCK_BYTES as usize; - assert_eq!(take(), vec![Ev::Map(7, client::PB_HEAP, size, 1), Ev::Append(7, client::PB_HEAP)]); - assert_cursor(&vm, 1); + let want: Vec = (0..chunks).map(|i| Ev::Map(7, chunk_gva(i), CHUNK_BYTES, i as u32 + 1)).collect(); + assert_eq!(take(), want); + assert_cursor(&f, &vm, chunks); // Mapping again maps nothing. f.map_pb(&vm).unwrap(); assert!(take().is_empty()); @@ -1101,6 +1119,8 @@ def consts(text, names): fn growth_backs_before_posting() { PARAM.with(|p| p.set(PARAM_DEFAULT)); let mut f = Firmware::new(); + let start = f.pools.ledger.backed as u64; + let first_chunks = f.pools.chunk_count(); f.fw.ask_after_reply = true; f.queue(); take(); @@ -1111,77 +1131,180 @@ def consts(text, names): assert!(passes <= 1000, "growth did not end"); } let log = take(); - let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); - assert_posted_backed(&log, 5, initial as u64); - // Chunks of at most 128 blocks, allocated only as needed: up to the maximum, no further. + assert_posted_backed(&log, 5, start); + // 16 MiB chunks, backed as the reserve needs them: up to the whole heap, no further. let allocs: Vec = log .iter() .filter_map(|e| if let Ev::Alloc(_, b) = e { Some(*b) } else { None }) .collect(); - assert_eq!(allocs.len() + 1, max_chunks(initial, max)); assert!(allocs.iter().all(|b| *b <= 16 << 20), "a growth chunk above 16 MiB"); assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 16 << 20); - assert_eq!(f.pools.backed(), max); + assert_eq!(allocs.len() + first_chunks, MAX_CHUNKS as usize); + assert_eq!(f.pools.ledger.backed, MAX_BLOCKS); + assert_eq!(f.pools.stats.waits, 0); + assert_eq!(f.pools.stats.refills, allocs.len() as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); - assert_cursor(&f.engine.flights[0].job.vm, f.pools.chunk_count()); + assert_cursor(&f, &f.engine.flights[0].job.vm, f.pools.chunk_count()); } - /// A failure while backing posts nothing and answers nothing; the request counter and the - /// VM's count stay; the next request is answered. - fn failure_posts_nothing(which: &'static std::thread::LocalKey>) -> Vec { + #[test] + fn allocation_follows_the_replies() { let mut f = Firmware::new(); - which.with(|c| c.set(1)); + // Below the reserve line: the pass's answers come first, then the chunk. + f.pools.ledger.posted = f.pools.ledger.backed - growth_reserve() + 1; f.queue(); - f.pass(); - let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_) | Ev::Sync)), 0, - "posted after a backing failure: {log:?}"); - assert_eq!((f.pools.ledger.posted, f.pools.ledger.next_request), (1024, 0), "ledger after a backing failure"); - assert_eq!((f.pools.stats.bad, f.pools.chunk_count()), (1, 1), "chunks after a backing failure"); - assert_cursor(&f.engine.flights[0].job.vm, 1); - assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -12")))); f.queue(); + take(); f.pass(); - let retry = take(); - assert!(retry.contains(&Ev::Producer(1034))); - assert_eq!(f.pools.chunk_count(), 2); - assert_cursor(&f.engine.flights[0].job.vm, 2); - log + let log = take(); + let last_reply = log.iter().rposition(|e| matches!(e, Ev::Reply(_))).expect("replies"); + let first_alloc = log.iter().position(|e| matches!(e, Ev::Alloc(..))).expect("a refill"); + assert!(first_alloc > last_reply, "allocation before a reply: {log:?}"); + assert_eq!(count(&log, |e| matches!(e, Ev::Reply(_))), 2); + assert_eq!(f.pools.stats.refills, 1); + } + + fn growth_reserve() -> u32 { + crate::g15::pools::growth::RESERVE_BLOCKS } #[test] - fn allocation_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_ALLOC); - assert_eq!(count(&log, is_backing), 0); + fn dry_reserve_waits_and_keeps_order() { + let mut f = Firmware::new(); + let start = f.pools.ledger.backed as u64; + FAIL_ALLOC.with(|c| c.set(1000)); + let mut log = Vec::new(); + let mut waited = None; + for _ in 0..40 { + f.queue(); + f.pass(); + let pass = take(); + let replied = pass.iter().any(|e| matches!(e, Ev::Reply(_))); + log.extend(pass); + if !replied { + waited = Some(f.fw.next_counter - 1); + break; + } + } + assert_posted_backed(&log, 5, start); + let waited = waited.expect("no request waited"); + assert_eq!(f.pools.ledger.posted, f.pools.ledger.backed); + let queued: Vec = f.events.grow.0.iter().map(|q| q[buffer::grow_request::COUNTER]).collect(); + assert_eq!(queued, vec![waited], "waiting request lost"); + assert_eq!(f.pools.ledger.next_request, waited); + assert!(f.pools.stats.waits >= 1); + // The allocator recovers: this pass still waits, then backs a chunk; the next answers. + FAIL_ALLOC.with(|c| c.set(0)); + f.pass(); + let log = take(); + assert!(!log.iter().any(|e| matches!(e, Ev::Reply(_)))); + assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 1); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 1); + let mapped = f.pools.ledger.backed as u64; + f.pass(); + let log = take(); + let replies: Vec = log + .iter() + .filter_map(|e| if let Ev::Reply(r) = e { Some(r[grow_reply::COUNTER]) } else { None }) + .collect(); + assert_eq!(replies, vec![waited]); + assert!(f.events.grow.is_empty()); + assert_posted_backed(&log, 5, mapped); } #[test] - fn map_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_MAP); + fn refill_map_failure_drops_the_chunk() { + let mut f = Firmware::new(); + let chunks = f.pools.chunk_count(); + let backed = f.pools.ledger.backed; + f.pools.ledger.posted = backed - growth_reserve() + 1; + FAIL_MAP.with(|c| c.set(1)); + take(); + f.pass(); + let log = take(); + assert_eq!(f.pools.chunk_count(), chunks, "chunk kept after a failed map"); let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); - assert!(matches!(backing[..], [Ev::Alloc(2, _), Ev::Free(2)]), "{backing:?}"); + assert!(matches!(backing[..], [Ev::Alloc(id, _), Ev::Free(free)] if id == free), "{backing:?}"); + assert_eq!(f.pools.ledger.backed, backed); + assert!(log.iter().any(|e| matches!(e, Ev::Log("warn", m) if m.contains("backing chunk")))); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks); + // The next pass backs it. + f.pass(); + assert_eq!(f.pools.chunk_count(), chunks + 1); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks + 1); } + /// A pending re-initialization with nothing published (a reclaim, or a recovery that + /// failed the flights): the posted count is about to drop, so nothing is backed. #[test] - fn append_failure_posts_nothing() { - let log = failure_posts_nothing(&FAIL_APPEND); - let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); + fn pending_reinit_without_flights_backs_nothing() { + let mut f = Firmware::without_flights(); + f.pools.ledger.posted = f.pools.ledger.backed - growth_reserve() + 1; + f.recovery.reinit = true; + take(); + f.pass(); + assert_eq!(count(&take(), is_backing), 0, "backed during a pending re-initialization"); + f.recovery.reinit = false; + f.pass(); + assert_eq!(f.pools.stats.refills, 1); + } + + /// A pending re-initialization with only compute published (no render can ask for + /// growth): nothing is backed. + #[test] + fn pending_reinit_with_compute_only_backs_nothing() { + let mut f = Firmware::new(); + f.renders[0] = 0; + f.pools.ledger.posted = f.pools.ledger.backed; + f.recovery.reinit = true; + take(); + for _ in 0..4 { + f.pass(); + } + assert_eq!(count(&take(), is_backing), 0, "backed for a compute-only flight"); + assert_eq!(f.pools.stats.refills, 0); + } + + /// A render published before the re-initialization was set (the flights behind a fragment + /// kill) still asks for growth: with its reserve used up, the refill must still back + /// chunks, or the request waits until the deadline. + #[test] + fn pending_reinit_with_a_live_render_refills() { + let mut f = Firmware::new(); // one live render + let start = f.pools.ledger.backed as u64; + f.pools.ledger.posted = f.pools.ledger.backed; + f.recovery.reinit = true; + f.queue(); + take(); + for _ in 0..4 { + f.pass(); + } + let s = &f.pools.stats; assert!( - matches!(backing[..], [Ev::Alloc(2, _), Ev::Map(5, _, _, 2), Ev::Unmap(5, _), Ev::Free(2)]), - "{backing:?}" + f.events.grow.is_empty(), + "growth request stranded: posted={} backed={} waits={} refills={}", + f.pools.ledger.posted, + f.pools.ledger.backed, + s.waits, + s.refills ); + // The request waits one pass for the chunk; the grant's pass tops the reserve up. + assert_eq!((s.waits, s.refills, s.requests), (1, 2, 1)); + assert_posted_backed(&take(), 5, start); } #[test] fn transport_failure_keeps_everything() { let mut f = Firmware::new(); + let chunks = f.pools.chunk_count(); f.fw.reply_fails = true; f.queue(); f.pass(); let log = take(); assert!(log.contains(&Ev::Producer(1034))); - assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (1034, 2)); - assert_cursor(&f.engine.flights[0].job.vm, 2); + // The grant crossed the reserve line: one chunk backed after it. + assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (1034, chunks + 1)); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks + 1); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); } @@ -1191,7 +1314,8 @@ def consts(text, names): let a = f.engine.flights[0].job.vm.clone(); grow_to(&mut f, 1100); take(); - assert_eq!(f.pools.chunk_count(), 2); + let chunks = f.pools.chunk_count(); + assert_eq!(chunks, 10); // A's work retires; a new VM runs next: it maps every chunk first. f.engine.flights.clear(); let b = VmState::new(6); @@ -1199,68 +1323,37 @@ def consts(text, names): f.fw.ctx = 6; f.map_pb(&b).unwrap(); let log = take(); - assert_eq!(count(&log, |e| matches!(e, Ev::Append(6, _))), 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(6, ..))), chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 0); - assert_cursor(&b, 2); + assert_cursor(&f, &b, chunks); // B grows the backing; A, behind, catches up when it runs again. grow_to(&mut f, 1200); let log = take(); - assert_posted_backed(&log, 6, 1152); + assert_posted_backed(&log, 6, chunks as u64 * CHUNK_BLOCKS as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 0); + assert_eq!(f.pools.chunk_count(), chunks + 1); f.engine.flights.clear(); f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); f.map_pb(&a).unwrap(); - assert_cursor(&a, f.pools.chunk_count()); - assert_eq!(a.pb_chunks.load(Ordering::Relaxed), f.pools.chunk_count()); + assert_cursor(&f, &a, f.pools.chunk_count()); } #[test] - fn growth_catches_the_running_vm_up_first() { + fn partial_catch_up_keeps_the_aliases_made() { let mut f = Firmware::new(); - let a = f.engine.flights[0].job.vm.clone(); - // B grows the backing while A is idle. - f.engine.flights.clear(); - let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.map_pb(&b).unwrap(); - f.fw.ctx = 6; grow_to(&mut f, 1100); - // A runs again; its growth request maps what it lacks before backing more. - f.engine.flights.clear(); - f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); - f.fw.ctx = 5; - take(); - grow_to(&mut f, 1130); - let log = take(); - assert_eq!(a.vm.held.borrow().len(), f.pools.chunk_count(), "running VM not caught up"); - assert_cursor(&a, f.pools.chunk_count()); - assert_posted_backed(&log, 5, 1024); - } - - #[test] - fn partial_catch_up_counts_only_held_aliases() { - let mut f = Firmware::new(); - grow_to(&mut f, 1450); + let chunks = f.pools.chunk_count(); f.engine.flights.clear(); let b = VmState::new(6); f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); take(); // The third alias cannot be made: two held. - FAIL_MAP.with(|c| c.set(0)); crate::MAP_OK_BEFORE_FAIL.with(|c| c.set(2)); FAIL_MAP.with(|c| c.set(1)); assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_cursor(&b, 5); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2); - // The third alias is made but cannot be held: dropped, still two. - FAIL_APPEND.with(|c| c.set(1)); - assert_eq!(f.map_pb(&b), Err(ENOMEM)); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2, "count after a failed append"); - assert!(take().iter().any(|e| matches!(e, Ev::Unmap(6, _)))); - assert_cursor(&b, 5); + assert_cursor(&f, &b, 2); f.map_pb(&b).unwrap(); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 5); - assert_cursor(&b, 5); + assert_cursor(&f, &b, chunks); } /// The ledger reset of Pools::init(), not a full recovery. @@ -1276,7 +1369,7 @@ def consts(text, names): let log = take(); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..) | Ev::Map(..))), 0); assert_eq!(f.pools.chunk_count(), chunks); - assert_posted_backed(&log, 5, f.pools.backed() as u64); + assert_posted_backed(&log, 5, f.pools.ledger.backed as u64); } #[test] @@ -1294,33 +1387,11 @@ def consts(text, names): #[test] fn catch_up_refuses_flights_of_another_vm() { let mut f = Firmware::new(); - let a = f.engine.flights[0].job.vm.clone(); let b = VmState::new(6); f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); assert_eq!(f.map_pb(&b), Err(EPROTO), "catch-up with a flight of another VM"); - assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 0); + assert_eq!(b.heap.list.lock().len(), 0); assert_eq!(count(&take(), is_backing), 0); - // A count past the chunks. - f.engine.flights.pop(); - a.pb_chunks.store(f.pools.chunk_count() + 1, Ordering::Relaxed); - assert_eq!(f.map_pb(&a), Err(EPROTO), "catch-up with a count past the chunks"); - } - - #[test] - fn grant_refuses_flights_of_two_vms() { - let mut f = Firmware::new(); - let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.renders.push(1); - f.queue(); - f.pass(); - let log = take(); - assert_eq!( - count(&log, |e| is_backing(e) || matches!(e, Ev::Reply(_) | Ev::Producer(_) | Ev::Record(..))), - 0, - "grant with flights of two VMs" - ); - assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -71")))); } #[test] @@ -1401,7 +1472,7 @@ def read(path, ref=None): raise SystemExit('embedded Ledger::new() differs from ' + OLD_REF) def run(mutation): - paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE) + paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE, VM) src = {path: read(path) for path in paths} if mutation: spec = {**MUTATIONS, **CONTROLS}[mutation] @@ -1411,15 +1482,23 @@ def run(mutation): src[path] = src[path].replace(old, new) pools, hw = src[POOLS], src[HW] bodies = [item(pools, s) for s in (' pub(crate) fn answer_growth(&mut self) -> Result {', - ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', + ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', ' pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result {', - ' fn running_vm(&self, ctx: u32) -> Result> {', - ' fn back_pb(&mut self, end: u32, ctx: u32) -> Result {')] + ' pub(crate) fn refill_backing(&mut self) {')] soft_max = item(pools, ' pub(crate) fn soft_max(&self) -> u32 {') - backed = item(pools, ' pub(crate) fn backed(&self) -> u32 {') + pools_methods = (item(pools, ' fn map_heap(&self, vm: &VmState) -> Result {') + '\n' + + item(pools, ' fn back_chunk(&mut self) -> Result> {')) chunk = item(pools, 'struct PbChunk {') + '\n' + item(pools, 'impl PbChunk {') + chunk_geometry = (re.search(r'^pub\(crate\) const CHUNK_BYTES: usize = [^;]+;', pools, re.M).group(0) + '\n' + + item(pools, 'pub(crate) const fn chunk_gva(i: usize) -> u64 {')) + vm_src = src[VM] + vm_methods = '\n'.join(item(vm_src, s) for s in ( + ' fn map_one(&self, heap: &mut KVec, i: usize, gem: ARef) -> Result {', + ' pub(crate) fn map_heap(&self, gems: impl Iterator>) -> Result {', + ' pub(crate) fn map_chunk(&self, i: usize, gem: ARef) -> Result {', + ' pub(crate) fn unmap_heap_above(&self, keep: usize) {')) start = pools.index(POOLS_NEW_BACKING[0]) - end = pools.index('\n', pools.index(POOLS_NEW_BACKING[1], start)) + 1 + end = pools.index(POOLS_NEW_BACKING[1], start) pools_new_backing = pools[start:end] if pools.count(PB_LEAF_ASSERT) != 1: raise SystemExit('pools.rs does not assert the PB leaf GPU-only') @@ -1453,7 +1532,8 @@ def run(mutation): control = consts(item(src[INITDATA], 'pub(crate) mod control {'), ['HALT_COUNT']) growth = src[GROWTH] clamp_fn = item(growth, 'const fn clamp(v: u32, lo: u32, hi: u32) -> u32 {') - tested = (''.join(bodies) + soft_max + backed + chunk + pools_new_backing + pb_max_blocks + growth + + tested = (''.join(bodies) + soft_max + pools_methods + chunk + chunk_geometry + vm_methods + + pools_new_backing + pb_max_blocks + growth + src[BUFFER] + sizes + tuning + pte + prot) dir = out / (mutation or 'actual') dir.mkdir(exist_ok=True) @@ -1466,7 +1546,8 @@ def run(mutation): f' #[path = "{dir / "buffer.rs"}"] pub mod buffer;\n' ' pub mod initdata { pub mod control {\n' + control + '\n} }\n' ' }\n' - f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth; }}\n' + f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth;\n' + ' use super::fw::buffer;\n use super::hw::client;\n' + chunk_geometry + '\n }\n' ' pub mod hw {\n' ' pub mod sizes {\n' + sizes + '\n}\n' ' pub mod tuning {\n' + tuning + '\n}\n' @@ -1499,19 +1580,20 @@ def run(mutation): ' use super::fw::buffer::{self, block_ctl};\n' ' use super::fw::channels::DevCtl;\n' ' use super::fw::initdata::control;\n' - ' use super::hw::{client, pte, tuning};\n' - ' use super::pools::growth::{self, Ledger};\n' + ' use super::hw::{self, client, pte, tuning};\n' + ' use super::pools::growth::{self, Answer, Ledger};\n' + ' use super::pools::{chunk_gva, CHUNK_BYTES};\n' ' use super::object::{self, HeldPages};\n' ' use crate::driver::AsahiDevice;\n' - ' use crate::{gem, mmu, ARef};\n' - ' use core::sync::atomic::{AtomicUsize, Ordering};\n' + ' use crate::{gem, mmu, ARef, Mutex};\n' ' use std::sync::Arc;\n' + MODEL.replace('p.RESET', reset).replace('POOLS_NEW_BACKING', '') + chunk + '\n' ' /// Pools::new\'s PB backing, verbatim.\n' ' pub fn pools_new_backing(dev: &AsahiDevice) -> Result<(Ledger, KVec)> {\n' + pools_new_backing + ' Ok((ledger, chunks))\n }\n' - ' impl Pools {\n' + soft_max + '\n' + backed + '\n }\n' + ' impl Pools {\n' + soft_max + '\n' + pools_methods + '\n }\n' + ' impl VmState {\n' + vm_methods + '\n }\n' ' impl Firmware {\n' + '\n'.join(bodies) + '\n }\n' ' }\n' '}\n' + TESTS) @@ -1551,7 +1633,8 @@ def no_core(): result = run(a.mutation) receipt = {'compiler': compiler, 'rustc_version': rustc_version, 'ref': a.ref, 'old_ref': OLD_REF, 'old_reference_checked_against_git': old_git is not None, **result, - 'scope': 'Actual answer_growth/answer_request/soft_max/pb_max_blocks, pools/growth.rs, ' + 'scope': 'Actual answer_growth/answer_request/map_pb/refill_backing/soft_max/pb_max_blocks, ' + 'the PB chunk and VM heap-mapping bodies, pools/growth.rs, ' 'fw/buffer.rs and fw/channels.rs over a model of the firmware objects with an ' 'ordered access log, and Ledger::new against its body at ' f'{OLD_REF[:12]}. Not kernel, firmware or GPU execution, timing, or recovery.'} From f637d677ebb708c22af3038d68074c20f756bced Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:56 +0530 Subject: [PATCH 03/46] drm/asahi: g15: m3: start the parameter buffer at 64 blocks The initial posting is 1024 blocks (128 MiB), with 9 chunks (144 MiB) backed when the GPU starts. On the J516C, an idle desktop with a 64-block start stayed at 64 posted blocks (8 MiB), and growth is a round trip of 1-2 ms per ten blocks. Post 64 blocks at (re)initialization by default, settable with g15_pb_blocks (64..4095, read-only after boot). The PB then needs two chunks (32 MiB) when the GPU starts: the posting and the reserve. tuning::PB_BLOCKS is removed. g15_pb_max_blocks, the firmware's growth maximum, is still raised to at least the initial posting; its help text no longer says that the 512 MiB heap is allocated in any case. This applies to the T6030 and the T6031, which share this code; it is untested on the T6030, and g15_pb_blocks=1024 restores the old start. The zero-words harness pins hw.rs whole; its hash is refreshed, and its T6030 configuration module stays pinned to the original bytes. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit b91c280b3294d2aa19a7f8d5796528b68412c6b6) [ryan: read g15_pb_blocks by value, as the Linux 7.3 module parameter API returns it, in the driver and in the host model] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/asahi_core.rs | 6 +- drivers/gpu/drm/asahi/g15/hw.rs | 7 +-- drivers/gpu/drm/asahi/g15/params.rs | 12 +++- drivers/gpu/drm/asahi/g15/pools.rs | 18 +++--- drivers/gpu/drm/asahi/g15/pools/growth.rs | 6 +- tools/asahi/g15-pb-growth-tests.py | 76 +++++++++++++++-------- tools/asahi/g15-zero-words-tests.py | 8 ++- 7 files changed, 85 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 42843e3459217..04a8d8cc4bb1e 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -170,7 +170,11 @@ module! { }, g15_pb_max_blocks: u32 { default: 3072, - description: "G15: parameter-buffer growth maximum given to the firmware, blocks of 128 KiB (1024..4095 in effect; the blocks above it, up to 4095, answer partial renders; the 512 MiB heap is allocated either way)", + description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", + }, + g15_pb_blocks: u32 { + default: 64, + description: "G15: PB blocks posted at (re)initialization, 128 KiB each (64..4095)", }, g15_early_tiling: u32 { default: 0, diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index d566a54953e07..c7f21a9110cee 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -571,11 +571,10 @@ pub(crate) mod tuning { pub(crate) const SLOT_ROTATE: bool = true; /// Flights per VM. pub(crate) const MAX_INFLIGHT: u32 = 2; - /// PB growth through event 7 / DEVCTL 7, initial and maximum blocks. + /// PB growth through event 7 / DEVCTL 7; the initial posting is `g15_pb_blocks` (64). pub(crate) const PB_GROW: bool = true; - /// Initial PB blocks, and the most that may be posted: one below the mapped heap's block - /// count, so the block-list producer (modulo K = 4096) never returns to 0. - pub(crate) const PB_BLOCKS: u32 = 1024; + /// The most PB blocks that may be posted: one below the mapped heap's block count, so the + /// block-list producer (modulo K = 4096) never returns to 0. pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS - 1; /// At the host deadline, arm the firmware kill. pub(crate) const HANG_PREEMPT: bool = true; diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index b1e3218fafc8d..3c2b60dbabeac 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -64,13 +64,19 @@ pub(crate) fn accept_text_prefix() -> u64 { } /// The parameter for the PB growth maximum the firmware is given, in blocks, clamped to -/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial 1024 blocks; -/// the blocks above it, up to the 4095 posted at most, answer the synchronous requests of -/// partial renders. It is read-only after boot and does not change the 512 MiB heap. +/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial posting; the +/// blocks above it, up to the 4095 posted at most, answer the synchronous requests of partial +/// renders. It is read-only after boot and does not change the heap's backing. pub(crate) fn pb_max_blocks() -> u32 { module_parameters::g15_pb_max_blocks.value().clamp(64, super::hw::tuning::PB_MAX_BLOCKS) } +/// The initial PB posting, in blocks (64..=4095; the ledger clamps it again). Read-only after +/// boot. +pub(crate) fn pb_blocks() -> u32 { + module_parameters::g15_pb_blocks.value().clamp(64, super::hw::tuning::PB_MAX_BLOCKS) +} + /// Whether FRAGMENT_BARRIERS is advertised and mapped to early tiling. pub(crate) fn early_tiling() -> bool { module_parameters::g15_early_tiling.value() != 0 diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index 15204f6091613..50f8501e7dfcf 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -12,9 +12,9 @@ //! the fixed client GVA: each VM's before its prefixes are published ([`Firmware::map_pb`]), //! and a chunk backed later also into the VM of the flights; nothing names its pages through //! the host window or context 0 (the records hold client GVAs only). Initial -//! [`super::hw::tuning::PB_BLOCKS`] (1024), at most [`super::hw::tuning::PB_MAX_BLOCKS`] -//! (4095) posted, so the producer never returns to 0; the firmware is given -//! [`Pools::soft_max`] (default 3072) as its growth maximum. Truthful K = 4096 in every +//! `g15_pb_blocks` (64), at most [`super::hw::tuning::PB_MAX_BLOCKS`] (4095) posted, so the +//! producer never returns to 0; the firmware is given [`Pools::soft_max`] (default 3072) as +//! its growth maximum. Truthful K = 4096 in every //! profile (PB reconstruction, also in recovery, synchronizes 8 * K bytes of the block list: //! K = 0x26cd overran the 0x8000-byte list and crashed the second recovery). //! - Backing: a request is granted from the backed blocks above the posted count; after the @@ -72,10 +72,10 @@ static_assert!(buffer::MAX_BLOCKS as u64 * buffer::BLOCK_BYTES == client::PB_HEA static_assert!(buffer::PAGE_LIST_SIZE as u64 == client::PB_PAGE_LIST_SIZE); static_assert!(uma_manager::SEEDS == sizes::FREELIST_SEED); static_assert!(uma_pools::POOL_SIZE == sizes::POOL_SIZE); -// The ledger keeps the driver's PB sizes: neither is capped by [`growth::MAX_POSTED`]. +// The ledger keeps the driver's PB maximum: it is not capped by [`growth::MAX_POSTED`]. The +// initial posting (`g15_pb_blocks`) is clamped to MIN_BLOCKS..=PB_MAX_BLOCKS. static_assert!(tuning::PB_MAX_BLOCKS <= growth::MAX_POSTED); -static_assert!(buffer::MIN_BLOCKS <= tuning::PB_BLOCKS); -static_assert!(tuning::PB_BLOCKS <= tuning::PB_MAX_BLOCKS); +static_assert!(buffer::MIN_BLOCKS <= tuning::PB_MAX_BLOCKS); // The PB alias is GPU-only (not cached and firmware-visible): dropping one is a plain unmap, // with no firmware cache flush (`KernelMapping::drop`). static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent()); @@ -373,7 +373,11 @@ impl Pools { true, )?; // Backing for the initial blocks and the reserve; growth adds the rest. - let mut ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let mut ledger = Ledger::new( + tuning::PB_GROW, + super::params::pb_blocks(), + tuning::PB_MAX_BLOCKS, + ); let mut chunks = KVec::with_capacity(growth::MAX_CHUNKS as usize, GFP_KERNEL)?; for _ in 0..Ledger::chunks_for(ledger.initial) { chunks.push(PbChunk::new(dev)?, GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 3ed143ee546ad..b94c5c58be733 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -257,11 +257,11 @@ mod tests { assert_eq!(Ledger::new(true, 512, 100).backed, 0); let fixed = Ledger::new(false, 512, 1024); assert_eq!((fixed.initial, fixed.max, fixed.posted), (4095, 4095, 4095)); - // The values the driver passes (tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS) are kept. - let l = Ledger::new(true, 1024, 4095); + // The values the driver passes (g15_pb_blocks 64, tuning::PB_MAX_BLOCKS) are kept. + let l = Ledger::new(true, 64, 4095); assert_eq!( (l.initial, l.max, l.posted, l.producer()), - (1024, 4095, 1024, 1024) + (64, 4095, 64, 64) ); } diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index c1e1f8998402d..e6a7137621080 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -5,9 +5,9 @@ The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), answer_request(), map_pb() and refill_backing(), Pools::soft_max(), map_heap() and back_chunk(), PbChunk, chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from -vm.rs VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() from -params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and -is_cached_noncoherent() from pgtable.rs; and the parameter's default from asahi_core.rs. It +vm.rs VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() and +pb_blocks() from params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and +is_cached_noncoherent() from pgtable.rs; and the parameters' defaults from asahi_core.rs. It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() and the DEVCTL wait, which logs every allocation, alias map, unmap, free, block record, barrier, producer @@ -186,7 +186,7 @@ 'three-refusals': ('default_growth_to_exhaustion', 'want Some(Log("warn"'), 'never-exhausted': ('default_growth_to_exhaustion', 'growth did not end'), 'warn-every-time': ('default_growth_to_exhaustion', 'later request after exhaustion'), - 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(1034)), want Some(Sync)'), + 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(74)), want Some(Sync)'), 'no-reply-barrier': ('default_growth_to_exhaustion', 'got Some(Reply([7, 1, 0, 5, 0, 256, 3, 1, 0, 0, 0, 0, 0, 0])), want Some(Sync)'), 'soft-boundary-inclusive': ('soft_max_boundaries', 'got Some(Log("info", "")), want Some(Log("dbg", ""))'), @@ -336,6 +336,7 @@ def consts(text, names): thread_local! { pub static LOG: RefCell> = RefCell::new(Vec::new()); pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); + pub static PARAM_BLOCKS: Cell = Cell::new(PB_BLOCKS_DEFAULT); /// Failures to inject: the next this many allocations or alias maps fail. pub static FAIL_ALLOC: Cell = Cell::new(0); pub static FAIL_MAP: Cell = Cell::new(0); @@ -384,6 +385,13 @@ def consts(text, names): } } pub static g15_pb_max_blocks: Param = Param; + pub struct Blocks; + impl Blocks { + pub fn value(&self) -> u32 { + crate::PARAM_BLOCKS.with(|p| p.get()) + } + } + pub static g15_pb_blocks: Blocks = Blocks; } pub mod driver { @@ -877,9 +885,10 @@ def consts(text, names): #[test] fn driver_values() { - let l = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let l = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); assert!(tuning::PB_GROW); - assert_eq!((l.initial, l.max, l.posted), (1024, 4095, 1024)); + assert_eq!(crate::PB_BLOCKS_DEFAULT, 64); + assert_eq!((l.initial, l.max, l.posted), (64, 4095, 64)); assert_eq!(PARAM_DEFAULT, 3072); PARAM.with(|p| p.set(PARAM_DEFAULT)); assert_eq!(params::pb_max_blocks(), 3072); @@ -892,8 +901,16 @@ def consts(text, names): for param in [0, 1, 63, 64, 65, 1023, 1024, 1025, 3071, 3072, 3073, 4094, 4095, 4096, 8192, u32::MAX] { PARAM.with(|p| p.set(param)); assert_eq!(params::pb_max_blocks(), clamp(param, 64, tuning::PB_MAX_BLOCKS), "pb_max_blocks, param {param}"); - // Below the initial 1024 the firmware still gets 1024. - assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}"); + // Below the initial 64 the firmware still gets 64. + assert_eq!(f.pools.soft_max(), clamp(param, 64, CAP), "soft_max, param {param}"); + } + // With a larger initial posting the firmware gets at least that. + crate::PARAM_BLOCKS.with(|b| b.set(1024)); + let f = Firmware::new(); + assert_eq!(f.pools.ledger.initial, 1024); + for param in [0, 64, 1023, 1024, 3072, 4095, u32::MAX] { + PARAM.with(|p| p.set(param)); + assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}, 1024 initial"); } } @@ -904,7 +921,7 @@ def consts(text, names): #[test] fn soft_max_boundaries() { - // 3074 is reached exactly (1024 + 205 * 10): that answer is not past it. + // 3074 is reached exactly (64 + 301 * 10): that answer is not past it. for param in [0, 1024, 3072, 3074, 3075, 4094, 4095, u32::MAX] { run_to_exhaustion(param); } @@ -961,7 +978,7 @@ def consts(text, names): assert_eq!(f.pools.stats.requests, 2, "requests taken in the pass"); assert_eq!(f.pools.stats.bad, 1); assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -5")))); - assert!(log.contains(&Ev::Producer(1024 + GROW_STEP))); + assert!(log.contains(&Ev::Producer(64 + GROW_STEP))); assert!(f.events.grow.is_empty()); } @@ -975,7 +992,7 @@ def consts(text, names): take(); f.pass(); let log = take(); - assert_eq!(f.pools.ledger.posted, 1024 + 3 * GROW_STEP); + assert_eq!(f.pools.ledger.posted, 64 + 3 * GROW_STEP); assert_eq!((f.pools.stats.grown, f.pools.stats.bad), (3 * GROW_STEP as u64, 0)); // Each failed reply is logged at info level. let info = log.iter().filter(|e| matches!(e, Ev::Log("info", m) if m.contains("DEVCTL 7: -5"))).count(); @@ -1008,7 +1025,7 @@ def consts(text, names): assert!(log.iter().all(|e| matches!(e, Ev::Log("warn", _))), "{log:?}"); assert_eq!(log.len(), 4); assert_eq!(f.pools.stats.bad, 4); - let mut fresh = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let mut fresh = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); fresh.backed = Ledger::chunks_for(fresh.initial) * CHUNK_BLOCKS; assert_eq!(f.pools.ledger, fresh); // No render published: not live. @@ -1091,7 +1108,7 @@ def consts(text, names): let log = take(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); let chunks = Ledger::chunks_for(initial) as usize; - assert_eq!(chunks, 9); + assert_eq!(chunks, 2); let want: Vec = (1..=chunks as u32).map(|id| Ev::Alloc(id, CHUNK_BYTES)).collect(); assert_eq!(log, want, "startup allocation"); assert_eq!((f.pools.ledger.backed, f.pools.chunk_count()), (chunks as u32 * CHUNK_BLOCKS, chunks)); @@ -1301,10 +1318,9 @@ def consts(text, names): f.queue(); f.pass(); let log = take(); - assert!(log.contains(&Ev::Producer(1034))); - // The grant crossed the reserve line: one chunk backed after it. - assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (1034, chunks + 1)); - assert_cursor(&f, &f.engine.flights[0].job.vm, chunks + 1); + assert!(log.contains(&Ev::Producer(74))); + assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (74, chunks)); + assert_cursor(&f, &f.engine.flights[0].job.vm, chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); } @@ -1312,10 +1328,10 @@ def consts(text, names): fn other_vms_catch_up_before_they_run() { let mut f = Firmware::new(); let a = f.engine.flights[0].job.vm.clone(); - grow_to(&mut f, 1100); + grow_to(&mut f, 200); take(); let chunks = f.pools.chunk_count(); - assert_eq!(chunks, 10); + assert_eq!(chunks, 3); // A's work retires; a new VM runs next: it maps every chunk first. f.engine.flights.clear(); let b = VmState::new(6); @@ -1327,7 +1343,7 @@ def consts(text, names): assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 0); assert_cursor(&f, &b, chunks); // B grows the backing; A, behind, catches up when it runs again. - grow_to(&mut f, 1200); + grow_to(&mut f, 300); let log = take(); assert_posted_backed(&log, 6, chunks as u64 * CHUNK_BLOCKS as u64); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 0); @@ -1341,7 +1357,7 @@ def consts(text, names): #[test] fn partial_catch_up_keeps_the_aliases_made() { let mut f = Firmware::new(); - grow_to(&mut f, 1100); + grow_to(&mut f, 200); let chunks = f.pools.chunk_count(); f.engine.flights.clear(); let b = VmState::new(6); @@ -1360,12 +1376,12 @@ def consts(text, names): #[test] fn ledger_reset_reuses_the_backing() { let mut f = Firmware::new(); - grow_to(&mut f, 1300); + grow_to(&mut f, 300); let chunks = f.pools.chunk_count(); take(); f.reset_ledger(); - assert_eq!(f.pools.ledger.posted, 1024); - grow_to(&mut f, 1300); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 300); let log = take(); assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..) | Ev::Map(..))), 0); assert_eq!(f.pools.chunk_count(), chunks); @@ -1423,7 +1439,7 @@ def consts(text, names): // Only inputs the old constructor took to the whole 4096-block heap changed. assert!(changed > 0 && changed < cases); // The driver's values are below the cap, so its ledger is the old one. - let d = (tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let d = (tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); let (old, new) = (OldLedger::new(d.0, d.1, d.2), Ledger::new(d.0, d.1, d.2)); if d.2 <= CAP { assert_eq!((old.initial, old.max, old.posted), (new.initial, new.max, new.posted)); @@ -1520,13 +1536,18 @@ def run(mutation): if init.count(line) != 1: raise SystemExit('Pools::init() does not hold: ' + line) reset = '\n'.join('p.' + line[len('self.'):] for line in LEDGER_RESET) - pb_max_blocks = item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> u32 {') + pb_max_blocks = (item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> u32 {') + '\n' + + item(src[PARAMS], 'pub(crate) fn pb_blocks() -> u32 {')) m = re.search(r'g15_pb_max_blocks: u32 \{\s*default: (\d+),', src[CORE]) if not m: raise SystemExit('g15_pb_max_blocks default not found') param_default = int(m.group(1)) + m = re.search(r'g15_pb_blocks: u32 \{\s*default: (\d+),', src[CORE]) + if not m: + raise SystemExit('g15_pb_blocks default not found') + blocks_default = int(m.group(1)) sizes = consts(item(hw, 'pub(crate) mod sizes {'), ['PB_BLOCKS']) - tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_BLOCKS', 'PB_MAX_BLOCKS']) + tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_MAX_BLOCKS']) client = consts(item(hw, 'pub(crate) mod client {'), ['PB_HEAP', 'PB_BLOCK_LIST']) pte = consts(item(hw, 'pub(crate) mod pte {'), ['LEAF_CACHED']) control = consts(item(src[INITDATA], 'pub(crate) mod control {'), ['HALT_COUNT']) @@ -1540,6 +1561,7 @@ def run(mutation): for path in (GROWTH, BUFFER, CHANNELS): (dir / Path(path).name).write_text(src[path]) code = (PREFIX + f'pub const PARAM_DEFAULT: u32 = {param_default};\n' + + f'pub const PB_BLOCKS_DEFAULT: u32 = {blocks_default};\n' + 'pub mod g15 {\n' ' pub mod fw {\n' f' #[path = "{dir / "channels.rs"}"] pub mod channels;\n' diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index 5b728e95fc03e..e659a507477b5 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -5,8 +5,10 @@ Uses synthetic public data and temporary host copies of production Rust modules. No GPU, firmware file, private fixture, kernel build or hardware access. The golden T6030 inputs below are from SiliconForest 9f09c85f8d3b, except the full -hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes. -Its T6030 configuration module is independently pinned to the original bytes. +hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes +and again when the initial PB posting became a module parameter (the tuning +constant PB_BLOCKS removed). Its T6030 configuration module is independently +pinned to the original bytes. """ import argparse @@ -22,7 +24,7 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "e4619590e670ed88d2a9bb76b03853081c615d3d0ad1c6a68321ce8e615ef8a0", + "hw.rs": "658c58026553c098b899aeffa0b6ae5be2b14921aef19df9e4f14789b7b890c4", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { From d2ad71374091ad10cce83fb99e33f0c29e7b9155 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:56 +0530 Subject: [PATCH 04/46] drm/asahi: g15: m3: reclaim the parameter buffer when the engine is idle Posted blocks never come back while a PB lives. The firmware recycles pages inside its own pool and the records give the host no view of it, so the posted count only grows until a recovery re-initializes the PB. On the J516C one dEQP case takes about 1200 blocks past the firmware's maximum each time it runs. The second run of one dEQP list exhausted the 512 MiB heap, and the GPU fail-stopped after the refusals. Re-initialize the PB when the engine has had nothing published and nothing pending for g15_pb_idle_ms (1000), with more than g15_pb_reclaim_blocks (1024) posted: - the next render prefix takes the re-initialization path of a recovered abort (fresh lists and manager, InitBM of the initial blocks), with no halt; - with g15_pb_reclaim 2 (the default), the chunks above those of the initial posting and the reserve are freed at once and unmapped from every VM of the registry; a VM that publishes later drops its stale aliases first (VmState::map_heap()); a chunk counts as freed only once every VM's alias of it is unmapped; - g15_pb_reclaim 1 re-initializes only, 0 never; - nothing is re-initialized when nothing grew since the initialization (posted == initial), whatever g15_pb_reclaim_blocks is. The idle clock is noted at the end of every poll pass and publication and when a job arrives. The check runs in idle poll passes and when a job arrives at an idle engine, before its publication, because an idle poll is not re-armed. This applies to the T6030 and the T6031, which share this code; it is untested on the T6030 (g15_pb_reclaim=0 turns it off, and g15_pb_blocks=1024 restores the old start). The debugfs pb: line appends reclaims and freed chunks. Host tests: the idle clock, the reclaim test of the ledger, a heavy scene freed and refilled, and (growth harness) a VM that maps chunks freed and backed again from their new objects. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit 54941febebcff7705d50698e24d244afb10034be) [ryan: read the three reclaim parameters by value, as the Linux 7.3 module parameter API returns them] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/asahi_core.rs | 12 ++ drivers/gpu/drm/asahi/g15/engine.rs | 7 ++ drivers/gpu/drm/asahi/g15/params.rs | 15 +++ drivers/gpu/drm/asahi/g15/poll.rs | 6 + drivers/gpu/drm/asahi/g15/pools.rs | 116 ++++++++++++++++++- drivers/gpu/drm/asahi/g15/pools/growth.rs | 134 +++++++++++++++++++++- drivers/gpu/drm/asahi/g15/vm.rs | 11 ++ tools/asahi/g15-pb-growth-tests.py | 48 +++++++- 8 files changed, 342 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 04a8d8cc4bb1e..6afd51ce4fa0a 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -176,6 +176,18 @@ module! { default: 64, description: "G15: PB blocks posted at (re)initialization, 128 KiB each (64..4095)", }, + g15_pb_reclaim: u32 { + default: 2, + description: "G15: PB reclaim when idle: 0 never, 1 re-initialize, 2 also free backing", + }, + g15_pb_idle_ms: u32 { + default: 1000, + description: "G15: idle time before a PB reclaim, ms (at least 100)", + }, + g15_pb_reclaim_blocks: u32 { + default: 1024, + description: "G15: PB blocks posted above which an idle engine reclaims the PB", + }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 51b3cfd10d594..6dd018369b478 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -515,8 +515,12 @@ impl Firmware { if self.engine.flights.capacity() < sizes::FLIGHTS { self.engine.flights.reserve(sizes::FLIGHTS, GFP_KERNEL)?; } + // A job arriving at an idle engine: the PB reclaim, if due, happens before its + // publication (idle polls are not re-armed, so this is where a long idle ends). + self.reclaim_check(); self.engine.pending.push(job.clone(), GFP_KERNEL)?; job.progress.set_pending(true); + self.note_idle(); let q = &job.queue; let owned = self.engine.queue_published(&job) + self @@ -601,6 +605,9 @@ impl Firmware { } } } + // A publication that left nothing published and nothing pending (jobs that failed + // before any visible record) starts the idle. + self.note_idle(); result } diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index 3c2b60dbabeac..7589e25aa4277 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -77,6 +77,21 @@ pub(crate) fn pb_blocks() -> u32 { module_parameters::g15_pb_blocks.value().clamp(64, super::hw::tuning::PB_MAX_BLOCKS) } +/// PB reclaim at idle: 0 never, 1 re-initialize, 2 re-initialize and free the backing. +pub(crate) fn pb_reclaim() -> u32 { + module_parameters::g15_pb_reclaim.value().min(2) +} + +/// Idle time before a PB reclaim, ns (at least 100 ms). +pub(crate) fn pb_idle_ns() -> u64 { + module_parameters::g15_pb_idle_ms.value().max(100) as u64 * 1_000_000 +} + +/// Posted blocks above which an idle engine reclaims the PB. +pub(crate) fn pb_reclaim_blocks() -> u32 { + module_parameters::g15_pb_reclaim_blocks.value() +} + /// Whether FRAGMENT_BARRIERS is advertised and mapped to early tiling. pub(crate) fn early_tiling() -> bool { module_parameters::g15_early_tiling.value() != 0 diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 566995eb342c9..e719e8e7ec3c2 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -207,6 +207,9 @@ impl Firmware { // prefixes published the retirement loop below drains first. if self.engine.flights.is_empty() { self.drain_rings()?; + // Idle bookkeeping of the PB: when the idle began, and the reclaim once it is due. + self.note_idle(); + self.reclaim_check(); if self.recovery_idle() { return Ok(false); } @@ -228,6 +231,9 @@ impl Firmware { self.engine.prof.polls += 1; self.engine.prof.poll_ns += now_ns().wrapping_sub(start); } + // The pass that retired the last flight is not re-armed: note the idle here, not only + // at the start of a pass that finds the flights empty. + self.note_idle(); published?; Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen) } diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index 50f8501e7dfcf..f9660a616c5d4 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -21,6 +21,12 @@ //! replies of a pass, while fewer than a reserve chunk remain, [`Firmware::refill_backing`] //! backs the next chunk, so a reply never waits for the allocator. Only a reserve that ran dry //! makes a request wait: it stays queued, the pass backs a chunk and the next pass answers it. +//! - Reclaim ([`Firmware::reclaim_check`], `g15_pb_reclaim`): posted blocks never come back +//! while a PB lives (the firmware recycles pages inside its own pool; the host has no view of +//! it), so an engine idle for `g15_pb_idle_ms` with more than `g15_pb_reclaim_blocks` posted +//! re-initializes the PB at its next render through the re-initialization path of a recovered +//! abort (fresh lists and manager, InitBM of the initial blocks, no halt) and frees the backing +//! above the kept chunks. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -62,7 +68,7 @@ use super::Firmware; use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{gem, mmu}; -pub(crate) use growth::{Answer, Ledger}; +pub(crate) use growth::{Answer, IdleClock, Ledger}; static_assert!(buffer::MAX_BLOCKS == sizes::PB_BLOCKS); static_assert!(buffer::MAX_PAGES == sizes::PB_PAGES); @@ -158,6 +164,9 @@ pub(crate) struct PbStats { pub(crate) waits: u64, /// Chunks backed after the GPU started. pub(crate) refills: u64, + /// Idle re-initializations, and the chunks they freed. + pub(crate) reclaims: u64, + pub(crate) freed: u64, } /// Render allocator and PB state: the allocators, the PB ledger and the spare set. @@ -185,6 +194,8 @@ pub(crate) struct Pools { c1_written: bool, /// The unanswered-at-the-maximum warning was given since the last initialization. exhausted_logged: bool, + /// The engine's idle clock (nothing published, nothing pending), for the reclaim. + idle: IdleClock, pub(crate) stats: PbStats, } @@ -403,6 +414,7 @@ impl Pools { ledger, c1_written: false, exhausted_logged: false, + idle: IdleClock::default(), stats: PbStats::default(), }) } @@ -539,6 +551,15 @@ impl Pools { vm.map_heap(self.chunks.iter().map(PbChunk::gem)) } + /// Frees the backing chunks from index `keep` on (their VM mappings are dropped by the + /// caller); the RAM returns when the last mapping of a chunk is gone. + fn free_chunks(&mut self, keep: usize) -> usize { + let freed = self.chunks.len().saturating_sub(keep); + self.chunks.truncate(keep); + self.ledger.unback_to(self.chunks.len() as u32); + freed + } + /// Backs one more chunk; the caller maps it where needed before the ledger posts from it. fn back_chunk(&mut self) -> Result> { if self.chunks.len() >= growth::MAX_CHUNKS as usize { @@ -551,11 +572,11 @@ impl Pools { } /// The debugfs line `pb: blocks initial/posted/firmware-max/max backed blocks/chunks - /// requests grown refused bad waits refills` (new keys are appended). + /// requests grown refused bad waits refills reclaims freed` (new keys are appended). pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { writeln!( f, - "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {} waits {} refills {}", + "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {} waits {} refills {} reclaims {} freed {}", self.ledger.initial, self.ledger.posted, self.soft_max(), @@ -567,7 +588,9 @@ impl Pools { self.stats.refused, self.stats.bad, self.stats.waits, - self.stats.refills + self.stats.refills, + self.stats.reclaims, + self.stats.freed ) } } @@ -660,6 +683,91 @@ impl Firmware { } } + /// Notes the engine idle (nothing published, nothing pending) or busy; the idle time is + /// what [`Self::reclaim_check`] measures. Called at the end of every poll pass and + /// publication (the pass that retires the last flight is not re-armed) and at a job's + /// arrival. + pub(crate) fn note_idle(&mut self) { + let idle = self.engine.flights.is_empty() && self.engine.pending.is_empty(); + self.pools.idle.note(idle, super::engine::now_ns()); + } + + /// Reclaims the PB when the engine has been idle for `g15_pb_idle_ms` with more than + /// `g15_pb_reclaim_blocks` posted: the next render prefix re-initializes the render + /// allocators and the PB (InitBM of the initial blocks; `recovery.reinit`, as after a + /// recovered abort but with no halt), and with `g15_pb_reclaim` 2 the backing chunks above + /// those of the initial posting and the reserve are freed now, unmapped from every VM of the + /// registry (a VM that publishes later drops its stale mappings at [`VmState::map_heap`]). + /// Called under the device lock when a job arrives at an idle engine and in idle poll + /// passes. + pub(crate) fn reclaim_check(&mut self) { + let level = super::params::pb_reclaim(); + if level == 0 + || !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || self.recovery.reinit + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + let floor = super::params::pb_reclaim_blocks(); + if !self + .pools + .ledger + .reclaim_due(floor, idle_ns, super::params::pb_idle_ns()) + { + return; + } + self.recovery.reinit = true; + self.pools.stats.reclaims += 1; + let posted = self.pools.ledger.posted; + let mut freed = 0; + if level >= 2 { + freed = self.free_backing(); + } + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted: re-initialization to {} at the next render, {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + self.pools.ledger.initial, + freed, + self.pools.chunks.len() + ); + } + + /// Frees the backing chunks above those a re-initialized heap keeps + /// ([`growth::Ledger::keep_chunks`]) and drops their mappings in every VM that could + /// publish; returns the chunks freed. If the VMs cannot be walked, their mappings keep the + /// pages until each VM drops its stale aliases ([`VmState::map_heap`]): nothing is counted + /// freed. + fn free_backing(&mut self) -> usize { + let keep = self.pools.ledger.keep_chunks() as usize; + let dropped = self.pools.free_chunks(keep); + // The mappings hold the chunks' pages: drop them in every VM that could publish. + let freed = match self.vms.open() { + Ok(states) => { + for state in states.iter() { + state.unmap_heap_above(keep); + } + dropped + } + Err(e) => { + dev_warn!( + self.dev(), + "PB heap: reclaim could not walk the VMs: {}\n", + e.to_errno() + ); + 0 + } + }; + self.pools.stats.freed += freed as u64; + freed + } + /// (Re)initializes the render UMA managers, PB lists and PB manager. `rebuild`: the priming /// prefix after a recovered abort: the first one moves to the uncached spare set, later /// ones rewrite it in place. diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index b94c5c58be733..1790699e2d7ac 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -18,7 +18,8 @@ //! whenever fewer than [`RESERVE_BLOCKS`] backed blocks remain above the posted count, so that //! it can be backed after a reply; a request that finds no backed block waits //! ([`Answer::Wait`]) and keeps its counter. A re-initialization ([`Ledger::reinit`]) returns -//! the posted count to the initial size and keeps the backing. +//! the posted count to the initial size and keeps the backing; [`Ledger::keep_chunks`] gives the +//! chunks that the initial posting and the reserve need. //! //! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. @@ -121,6 +122,11 @@ impl Ledger { self.backed += CHUNK_BLOCKS; } + /// Drops the backing to `chunks` chunks. + pub(crate) const fn unback_to(&mut self, chunks: u32) { + self.backed = chunks * CHUNK_BLOCKS; + } + /// Backed blocks above the posted count. pub(crate) const fn spare(&self) -> u32 { self.backed.saturating_sub(self.posted) @@ -142,6 +148,18 @@ impl Ledger { } } + /// Chunks a re-initialized heap keeps: those of the initial posting and the reserve. + pub(crate) const fn keep_chunks(&self) -> u32 { + Self::chunks_for(self.initial) + } + + /// Whether the heap is to be re-initialized: idle for `idle_ns` with more than `floor` + /// blocks posted, and grown since the initialization (with nothing grown a + /// re-initialization posts the same blocks again). + pub(crate) const fn reclaim_due(&self, floor: u32, idle_ns: u64, min_idle_ns: u64) -> bool { + self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -203,6 +221,37 @@ impl Ledger { } } +/// The engine's idle clock for the reclaim policy: when nothing was published and nothing +/// pending, from the pass that retired the last flight on. Noted at the end of every poll pass +/// and publication (a pass that empties the flights is not re-armed, so a note only at the +/// start of a pass with empty flights depends on a backup pass happening to run) and, busy, +/// when a job is enqueued. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct IdleClock { + /// When the idle began; 0 while busy. + since_ns: u64, +} + +impl IdleClock { + /// Notes the engine idle or busy at `now_ns`; an idle already begun keeps its start. + pub(crate) fn note(&mut self, idle: bool, now_ns: u64) { + if !idle { + self.since_ns = 0; + } else if self.since_ns == 0 { + self.since_ns = now_ns.max(1); + } + } + + /// The idle time at `now_ns`; 0 while busy. + pub(crate) fn idle_ns(&self, now_ns: u64) -> u64 { + if self.since_ns == 0 { + 0 + } else { + now_ns.wrapping_sub(self.since_ns) + } + } +} + /// The halt epoch a request names (u64 at words 5..6). pub(crate) const fn request_epoch(request: &[u32; 14]) -> u64 { request[grow_request::EPOCH] as u64 | (request[grow_request::EPOCH + 1] as u64) << 32 @@ -401,6 +450,7 @@ mod tests { assert_eq!(Ledger::chunks_for(4095), MAX_CHUNKS); assert_eq!(MAX_CHUNKS, 32); let mut l = Ledger::new(true, 64, 4095); + assert_eq!(l.keep_chunks(), 2); assert_eq!(l.spare(), 0); assert!(l.refill_due()); // Nothing backed: a request waits and the counter does not advance. @@ -449,9 +499,11 @@ mod tests { ); l.post(5); assert_eq!(l.accept(&request(3, 3, 1)), Answer::Refuse); - // Re-initialization keeps the backing. + // Re-initialization keeps the backing; dropping it to the kept chunks. l.reinit(); assert_eq!((l.posted, l.backed), (64, 4096)); + l.unback_to(l.keep_chunks()); + assert_eq!(l.backed, 256); } /// Every ledger the constructor makes, the driver's included, backed a chunk at a time @@ -492,6 +544,84 @@ mod tests { } } + #[test] + fn reclaim() { + let mut l = Ledger::new(true, 64, 4095); + let s = 1_000_000_000; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1024; + assert!(!l.reclaim_due(1024, 5 * s, s)); + l.posted = 1025; + assert!(l.reclaim_due(1024, s, s)); + assert!(!l.reclaim_due(1024, s - 1, s)); + // The clock: busy is 0, an idle keeps its start, busy ends it. + let mut c = IdleClock::default(); + assert_eq!(c.idle_ns(10 * s), 0); + c.note(true, 10 * s); + c.note(true, 12 * s); + assert_eq!(c.idle_ns(15 * s), 5 * s); + c.note(false, 16 * s); + assert_eq!(c.idle_ns(20 * s), 0); + // A clock started at 0 ns still counts as idle. + c.note(true, 0); + assert_eq!(c.idle_ns(s), s - 1); + // Nothing grown since the initialization: no re-initialization, whatever the floor. + let l = Ledger::new(true, 64, 4095); + assert!( + !l.reclaim_due(32, 5 * s, s), + "re-initialization with nothing grown" + ); + } + + /// A heavy scene, a reclaim that frees the backing, refills during the next heavy scene, + /// an idle gap with a small job in it, and the next render must reclaim again. + #[test] + fn reclaim_after_free_then_refill() { + let s = 1_000_000_000; + let (floor, min_idle) = (1024, s); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + let mut clock = IdleClock::default(); + // A heavy scene grows the PB to 1484 blocks, refilling as it goes. + let heavy = |l: &mut Ledger, clock: &mut IdleClock, t: u64| { + clock.note(false, t); + let mut counter = l.next_request; + while l.posted < 1484 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + // The pass that retires the last flight notes the idle; no later pass may run. + clock.note(true, t + s); + }; + heavy(&mut l, &mut clock, 100 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // 5 s later a small job arrives: the reclaim is due; it frees the backing above the + // kept chunks; the re-initialization happens at the next render. + let t = 106 * s; + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + l.unback_to(l.keep_chunks()); + clock.note(false, t); + assert_eq!(l.backed, 2 * CHUNK_BLOCKS); + clock.note(true, t + s / 2); + // The next render re-initializes, then the second heavy scene refills to 1484 again. + l.reinit(); + assert_eq!((l.posted, l.backed), (64, 256)); + heavy(&mut l, &mut clock, 110 * s); + assert_eq!((l.posted, l.backed), (1484, 13 * CHUNK_BLOCKS)); + // The same gap again: the reclaim must be due at the small job's arrival. + let t = 116 * s; + assert_eq!(clock.idle_ns(t), 5 * s); + assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); + } + #[test] fn grants_only_backed_blocks() { let mut l = Ledger::new(true, 64, 4095); diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index 884111d282bdd..260f5e6b2cbb1 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -321,6 +321,17 @@ impl Registry { .ok_or(ENOENT) } + /// The VMs whose client handle is open (and the closed ones not pruned yet), for a walk + /// that takes VM locks: not under the registry lock. + pub(crate) fn open(&self) -> Result>> { + let vms = self.vms.lock(); + let mut out = KVec::with_capacity(vms.len(), GFP_KERNEL)?; + for state in vms.iter() { + out.push(state.clone(), GFP_KERNEL)?; + } + Ok(out) + } + /// Latches `error` on every VM whose client handle is open. The fail-stop path calls it under /// the device lock. pub(crate) fn fail_all(&self, error: Error) { diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index e6a7137621080..124cd66502d02 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -37,7 +37,8 @@ render is published, and still backs for a render published before it; a failed reply keeps everything; - a VM's aliases are the chunks, contiguous from the heap's start; a VM that runs after another - grew the backing maps it first (at publication); a partial catch-up keeps the aliases made; + grew the backing maps it first (at publication); a partial catch-up keeps the aliases made; a + chunk freed and backed again (an idle reclaim) is unmapped and mapped again at publication; - the one-VM rule: a catch-up with a flight of another VM is refused (EPROTO) before anything is mapped; - the ledger reset of Pools::init() re-posts from the backing without allocating; @@ -166,6 +167,8 @@ # Refill for any live flight, also compute-only ones that cannot ask for growth. 'refill-for-any-flight': (POOLS, 'if self.recovery.reinit && !render {', 'if self.recovery.reinit && self.engine.flights.is_empty() {'), + # Keep a VM's alias of a chunk that was freed and backed again. + 'map-without-identity': (VM, ' if core::ptr::eq(&*have.gem, &*gem) {', ' if true {'), # Catch up without the one-VM check. 'no-activation-guard': (POOLS, '.filter(|f| !core::ptr::eq(&*f.job.vm, vm))', '.filter(|_| false)'), # Back the whole heap when the GPU starts. @@ -200,6 +203,7 @@ 'refill-during-reinit': ('pending_reinit_without_flights_backs_nothing', 'backed during a pending re-initialization'), 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), + 'map-without-identity': ('freed_chunks_are_mapped_again', 'alias of another object'), 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), @@ -578,6 +582,12 @@ def consts(text, names): pub fn chunk_capacity(&self) -> usize { self.chunks.capacity() } + /// What a reclaim does to the backing (the PB re-initialization included). + pub fn drop_chunks(&mut self, keep: usize) { + self.chunks.truncate(keep); + self.ledger.reinit(); + self.ledger.unback_to(keep as u32); + } /// The backing objects' ids, in heap order. pub fn chunk_ids(&self) -> Vec { self.chunks.iter().map(|c| c.backing.gem.id).collect() @@ -1400,6 +1410,42 @@ def consts(text, names): assert_eq!(f.pools.stats.bad, 1); } + /// What a reclaim leaves behind in a VM that was not walked (a closed VM not pruned yet, or + /// one created meanwhile): its aliases of freed chunks go, and chunks backed again at the + /// same GVA are mapped from their new objects at its next publication. + #[test] + fn freed_chunks_are_mapped_again() { + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 300); + let chunks = f.pools.chunk_count(); + assert!(chunks > 2); + // The reclaim frees the chunks above the kept two; A is not walked. + f.engine.flights.clear(); + f.pools.drop_chunks(2); + take(); + // B runs and grows the backing again: new objects at the old GVAs. + let b = VmState::new(6); + f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); + f.fw.ctx = 6; + f.map_pb(&b).unwrap(); + grow_to(&mut f, 300); + assert_eq!(f.pools.chunk_count(), chunks); + // A runs again: its stale aliases are replaced by the new objects'. + f.engine.flights.clear(); + f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); + take(); + f.map_pb(&a).unwrap(); + let log = take(); + assert_cursor(&f, &a, chunks); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), chunks - 2); + // With fewer chunks than A maps, the ones past the backing go too. + f.pools.drop_chunks(2); + f.map_pb(&a).unwrap(); + assert_cursor(&f, &a, 2); + } + #[test] fn catch_up_refuses_flights_of_another_vm() { let mut f = Firmware::new(); From 330a11a0d922a0901d8d2faa91056e3cea73e61c Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:57 +0530 Subject: [PATCH 05/46] drm/asahi: g15: m3: re-initialize the parameter buffer at a client boundary The idle reclaim needs g15_pb_idle_ms without work. Heavy scenes started back to back, each by a new client a few hundred ms after the last, never reach it. On the J516C three such scenes of about 1420 blocks each ran the heap to its maximum, and the fourth hung the GPU into fail-stop. Re-initialize also after 50 ms idle with more than half the maximum posted and grown since the initialization (an initial posting above half the maximum is not drained). 50 ms is longer than the gaps between one client's renders. On the J516C a re-initialization with no idle time faulted at the heap base, at a client switch, the second time in a row with no recovery between; with 50 ms of idle none faulted, also between two renders of one client 60 ms apart. This path keeps the backing, so the next scene does not allocate it again. The idle reclaim now also runs when more chunks are backed than the posted blocks need (free_due()), so backing kept this way is freed even if the next client posts below g15_pb_reclaim_blocks. With no render after the client boundary, the re-initialization stays pending, and with g15_pb_reclaim 2 the check frees the backing above the kept chunks once the engine has been idle for g15_pb_idle_ms (a recovery's pending re-initialization still stops the check). This applies to the T6030 and the T6031, which share this code; it is untested on the T6030 (g15_pb_reclaim=0 turns it off, and g15_pb_blocks=1024 restores the old start). Host tests: back-to-back heavy scenes never reach the maximum and keep their backing, and kept backing is freed at the next long idle. The growth harness also runs reclaim_check(), free_backing() and note_idle() over a clock: a heavy client that exits with nothing rendered after it is re-initialized after 50 ms and its kept backing freed after the long idle, its posted count kept until the InitBM; at level 1 the backing stays; a recovery's pending re-initialization frees nothing; nothing grown, nothing is re-initialized. The reclaim keeps the posted count until the next render's InitBM, as the driver does: the same client rendering again after 60 ms grows again from the kept backing with nothing allocated, after 1.1 s over freshly backed chunks mapped before they are posted, and a compute-only submission between the reclaim and the InitBM maps, backs and posts nothing. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit 59aced969cbefdc9ac2801276c15e91c97de6065) [ryan: the host model reads the reclaim parameters by value, as the Linux 7.3 module parameter API returns them] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/pools.rs | 60 ++- drivers/gpu/drm/asahi/g15/pools/growth.rs | 115 ++++++ tools/asahi/g15-pb-growth-tests.py | 470 ++++++++++++++++++++-- 3 files changed, 597 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index f9660a616c5d4..f03b196760b6a 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -26,7 +26,10 @@ //! it), so an engine idle for `g15_pb_idle_ms` with more than `g15_pb_reclaim_blocks` posted //! re-initializes the PB at its next render through the re-initialization path of a recovered //! abort (fresh lists and manager, InitBM of the initial blocks, no halt) and frees the backing -//! above the kept chunks. +//! above the kept chunks (also with fewer posted, when more is backed than they need); an +//! engine idle for 50 ms (a client boundary) with more than half the maximum posted +//! re-initializes it the same way but keeps the backing; if no render follows, with +//! `g15_pb_reclaim` 2 the backing above the kept chunks is freed after `g15_pb_idle_ms`. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -196,6 +199,9 @@ pub(crate) struct Pools { exhausted_logged: bool, /// The engine's idle clock (nothing published, nothing pending), for the reclaim. idle: IdleClock, + /// The pending re-initialization was set by a reclaim ([`Firmware::reclaim_check`]), not by + /// a recovery; cleared by [`Self::init`]. + reclaim_pending: bool, pub(crate) stats: PbStats, } @@ -415,6 +421,7 @@ impl Pools { c1_written: false, exhausted_logged: false, idle: IdleClock::default(), + reclaim_pending: false, stats: PbStats::default(), }) } @@ -476,6 +483,7 @@ impl Pools { // capacity. self.ledger.reinit(); self.exhausted_logged = false; + self.reclaim_pending = false; if tuning::PB_GROW { // C1 names the initial pages again. self.c1_written = false; @@ -693,19 +701,24 @@ impl Firmware { } /// Reclaims the PB when the engine has been idle for `g15_pb_idle_ms` with more than - /// `g15_pb_reclaim_blocks` posted: the next render prefix re-initializes the render - /// allocators and the PB (InitBM of the initial blocks; `recovery.reinit`, as after a - /// recovered abort but with no halt), and with `g15_pb_reclaim` 2 the backing chunks above - /// those of the initial posting and the reserve are freed now, unmapped from every VM of the - /// registry (a VM that publishes later drops its stale mappings at [`VmState::map_heap`]). - /// Called under the device lock when a job arrives at an idle engine and in idle poll - /// passes. + /// `g15_pb_reclaim_blocks` posted or more backing than the posted blocks need, or for + /// [`growth::DRAIN_IDLE_NS`] with more than half the maximum posted + /// ([`growth::Ledger::drain_reinit_due`]; re-initialization only): the next render prefix + /// re-initializes the render allocators and the PB (InitBM of the initial blocks; + /// `recovery.reinit`, as after a recovered abort but with no halt), and with + /// `g15_pb_reclaim` 2 the backing chunks above those of the initial posting and the reserve + /// are freed now, unmapped from every VM of the registry (a VM that publishes later drops + /// its stale mappings at [`VmState::map_heap`]). While a reclaim's re-initialization is + /// pending (a client-boundary one kept the backing), `g15_pb_reclaim` 2 frees the backing + /// above those chunks once the engine has been idle for `g15_pb_idle_ms` + /// ([`growth::Ledger::pending_free_due`]). Called under the device lock when a job arrives + /// at an idle engine and in idle poll passes. pub(crate) fn reclaim_check(&mut self) { let level = super::params::pb_reclaim(); if level == 0 || !self.engine.flights.is_empty() || !self.engine.pending.is_empty() - || self.recovery.reinit + || (self.recovery.reinit && !self.pools.reclaim_pending) || self.recovery.armed.is_some() || self.engine.frozen || self.recovery.failed().is_some() @@ -714,18 +727,35 @@ impl Firmware { } let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); let floor = super::params::pb_reclaim_blocks(); - if !self - .pools - .ledger - .reclaim_due(floor, idle_ns, super::params::pb_idle_ns()) - { + let min_idle = super::params::pb_idle_ns(); + let ledger = &self.pools.ledger; + if self.recovery.reinit { + if level >= 2 && ledger.pending_free_due(idle_ns, min_idle) { + let posted = ledger.posted; + let freed = self.free_backing(); + dev_info!( + self.dev(), + "PB heap: idle {} ms with {} blocks posted and the re-initialization pending: {} chunks freed, {} backed\n", + idle_ns / 1_000_000, + posted, + freed, + self.pools.chunks.len() + ); + } + return; + } + let idle = ledger.reclaim_due(floor, idle_ns, min_idle) + || (level >= 2 && ledger.free_due(idle_ns, min_idle)); + if !idle && !ledger.drain_reinit_due(idle_ns) { return; } self.recovery.reinit = true; + self.pools.reclaim_pending = true; self.pools.stats.reclaims += 1; let posted = self.pools.ledger.posted; let mut freed = 0; - if level >= 2 { + // A short idle keeps the backing: the next scene would allocate it again. + if level >= 2 && idle { freed = self.free_backing(); } dev_info!( diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index 1790699e2d7ac..c98aef060f94e 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -62,6 +62,10 @@ pub(crate) const RESERVE_BLOCKS: u32 = CHUNK_BLOCKS; /// Chunks of the whole heap. pub(crate) const MAX_CHUNKS: u32 = MAX_BLOCKS / CHUNK_BLOCKS; +/// The idle time before a re-initialization past half the maximum (a client boundary, not a +/// gap between one client's renders). +pub(crate) const DRAIN_IDLE_NS: u64 = 50_000_000; + /// The answer to an admitted growth request. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum Answer { @@ -160,6 +164,31 @@ impl Ledger { self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns } + /// Whether the backing is to be freed: idle for `idle_ns` with more chunks backed than the + /// posted blocks and the reserve need. A client-boundary re-initialization keeps the + /// backing of a heavy scene; if the next client posts below the reclaim floor, this frees it. + pub(crate) const fn free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS + } + + /// While a reclaim's re-initialization is pending (the posted count returns to the initial + /// size at the next render): whether the backing above [`Self::keep_chunks`] is to be + /// freed, idle for `min_idle_ns`. A client-boundary re-initialization keeps the backing; + /// with no render after it, this frees it. + pub(crate) const fn pending_free_due(&self, idle_ns: u64, min_idle_ns: u64) -> bool { + idle_ns >= min_idle_ns && self.backed > self.keep_chunks() * CHUNK_BLOCKS + } + + /// Whether a short idle re-initializes the heap: more than half the maximum posted and + /// idle for [`DRAIN_IDLE_NS`]. Back-to-back heavy scenes (a new client every few hundred + /// ms) never reach `g15_pb_idle_ms`, and posted blocks never come back while a PB lives, so + /// without this they run the heap to the maximum. A re-initialization with no idle time + /// faulted at the heap base (at a client switch, the second in a row with no recovery + /// between); with 50 ms of idle none did, also between two renders of one client. + pub(crate) const fn drain_reinit_due(&self, idle_ns: u64) -> bool { + self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -573,6 +602,77 @@ mod tests { ); } + /// Heavy scenes of ~1420 blocks each, new clients back to + /// back. The engine re-initializes at a client boundary once past half the maximum, so the + /// heap never reaches it; the backing stays (no refill on the next scene). + #[test] + fn back_to_back_heavy_scenes() { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + let mut counter = 0; + for _ in 0..10 { + if l.drain_reinit_due(DRAIN_IDLE_NS) { + l.reinit(); + counter = l.next_request; + } + let start = l.posted; + while l.posted < start + 1420 { + while l.refill_due() { + l.back_chunk(); + } + match l.accept(&request(1, counter, 0)) { + Answer::Grant { count, .. } => { + l.post(count); + } + other => panic!("{other:?} at {}", l.posted), + } + counter += 1; + } + assert!(l.posted < l.max); + } + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + assert_eq!(l.backed, 24 * CHUNK_BLOCKS); + l.posted = 2047; + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS)); + l.posted = 2048; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // Not between two renders of one client. + assert!(!l.drain_reinit_due(DRAIN_IDLE_NS - 1)); + assert!(!l.drain_reinit_due(0)); + // An initial posting above half the maximum: nothing grew, nothing to drain. + let l = Ledger::new(true, 3000, 4095); + assert!( + !l.drain_reinit_due(DRAIN_IDLE_NS), + "drain with nothing grown" + ); + } + + /// A client-boundary re-initialization keeps the backing + /// of a heavy scene, the next client posts little; once idle, the backing above what the + /// posted blocks need is due to be freed although the posted count is below the floor. + #[test] + fn kept_backing_freed_at_idle() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(!l.free_due(5 * s, s)); + l.reinit(); + l.posted = 264; + assert!(!l.reclaim_due(1024, 5 * s, s)); + assert!(!l.free_due(s - 1, s)); + assert!(l.free_due(s, s)); + // After the re-initialization and the free, nothing more is due. + l.reinit(); + l.unback_to(l.keep_chunks()); + assert!(!l.free_due(5 * s, s)); + // A heap backed for what it posts plus the reserve is not freed. + l.posted = 1000; + l.unback_to(Ledger::chunks_for(1000)); + assert!(!l.free_due(5 * s, s)); + } + /// A heavy scene, a reclaim that frees the backing, refills during the next heavy scene, /// an idle gap with a small job in it, and the next render must reclaim again. #[test] @@ -622,6 +722,21 @@ mod tests { assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); } + /// A client-boundary re-initialization with 23 chunks backed and no render after it: the + /// kept backing is due to be freed after the long idle, and once freed nothing more is due. + #[test] + fn pending_reinit_frees_kept_backing() { + let s = 1_000_000_000; + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 2800; + assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + assert!(!l.pending_free_due(s - 1, s)); + assert!(l.pending_free_due(s, s)); + l.unback_to(l.keep_chunks()); + assert!(!l.pending_free_due(100 * s, s)); + } + #[test] fn grants_only_backed_blocks() { let mut l = Ledger::new(true, 64, 4095); diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 124cd66502d02..43949e9d91be8 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -3,10 +3,11 @@ """Run the G15 PB growth answer and its on-demand backing, actual bodies, against a model. The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), -answer_request(), map_pb() and refill_backing(), Pools::soft_max(), map_heap() and -back_chunk(), PbChunk, chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from -vm.rs VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() and -pb_blocks() from params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and +answer_request(), map_pb(), refill_backing(), note_idle(), reclaim_check() and free_backing(), +Pools::soft_max(), map_heap(), free_chunks() and back_chunk(), PbChunk, +chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from vm.rs +VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks(), +pb_blocks(), pb_reclaim(), pb_idle_ns() and pb_reclaim_blocks() from params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and is_cached_noncoherent() from pgtable.rs; and the parameters' defaults from asahi_core.rs. It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() and the DEVCTL @@ -41,6 +42,20 @@ chunk freed and backed again (an idle reclaim) is unmapped and mapped again at publication; - the one-VM rule: a catch-up with a flight of another VM is refused (EPROTO) before anything is mapped; +- the idle reclaim, actual reclaim_check(), free_backing() and note_idle() over a clock: a heavy + client that exits with nothing rendered after it is re-initialized after 50 ms with the backing + kept, and the kept backing is freed after the long idle, its posted count kept until the InitBM; + at reclaim level 1 the backing stays; a recovery's pending re-initialization frees nothing; +- nothing grown since the initialization (an initial posting above half the maximum, or a + reclaim floor below it): no re-initialization; +- the actual reclaim keeps the posted count until the next render's InitBM (a model of the + publication: map_pb() first, then for a render prefix with a re-initialization pending the + ledger reset of Pools::init()): the same client rendering again after 60 ms re-posts the + initial blocks and grows again from the kept backing with nothing allocated; after 1.1 s it + grows again over freshly backed chunks, each mapped before it is posted; a compute-only + submission between the reclaim and the InitBM maps, backs and posts nothing; +- a reclaim whose walk of the VMs fails counts nothing freed; the VM's next publication drops + its stale aliases and releases the pages; - the ledger reset of Pools::init() re-posts from the backing without allocating; - the PB leaf decodes as GPU-only, not cached-noncoherent: dropping an alias is a plain unmap; - Ledger::new() is Ledger::new() as of dba2a110de00 (the carried upstream constructor), capped at @@ -109,7 +124,8 @@ # The two statements of Pools::init() that reset the ledger. LEDGER_RESET = ['self.ledger.reinit();', - 'self.exhausted_logged = false;'] + 'self.exhausted_logged = false;', + 'self.reclaim_pending = false;'] MUTATIONS = { # Answer until the queue is empty, including requests queued by the replies' waits. @@ -169,6 +185,47 @@ 'if self.recovery.reinit && self.engine.flights.is_empty() {'), # Keep a VM's alias of a chunk that was freed and backed again. 'map-without-identity': (VM, ' if core::ptr::eq(&*have.gem, &*gem) {', ' if true {'), + # Stop the reclaim check on a reclaim's pending re-initialization (the kept backing stays). + 'pending-reinit-stops-free': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + # Free the kept backing at reclaim level 1. + 'pending-free-at-level-1': (POOLS, 'if level >= 2 && ledger.pending_free_due(idle_ns, min_idle) {', + 'if ledger.pending_free_due(idle_ns, min_idle) {'), + # Do not record that the pending re-initialization is a reclaim's. + 'reclaim-pending-not-set': (POOLS, ' self.pools.reclaim_pending = true;\n', ''), + # Free for any pending re-initialization, also a recovery's. + 'free-on-recovery-reinit': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + # Re-initialize at an idle with nothing grown (posted above the floor only). + 'reclaim-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns', + 'self.posted > floor && idle_ns >= min_idle_ns'), + # Re-initialize at a client boundary with nothing grown (initial above half the maximum). + 'drain-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS', + 'self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS'), + # Reset the posted count at the idle reclaim instead of at the next render's InitBM. + 'reinit-at-idle-reclaim': (POOLS, ' if level >= 2 && idle {\n freed = self.free_backing();\n', + ' if level >= 2 && idle {\n self.pools.ledger.reinit();\n freed = self.free_backing();\n'), + # Reset the posted count at any reclaim instead of at the next render's InitBM. + 'reinit-at-any-reclaim': (POOLS, ' self.pools.reclaim_pending = true;\n', + ' self.pools.reclaim_pending = true;\n self.pools.ledger.reinit();\n'), + # Count the chunks freed although the VMs could not be walked. + 'freed-without-walk': (POOLS, ' );\n 0\n }\n };\n', + ' );\n dropped\n }\n };\n'), # Catch up without the one-VM check. 'no-activation-guard': (POOLS, '.filter(|f| !core::ptr::eq(&*f.job.vm, vm))', '.filter(|_| false)'), # Back the whole heap when the GPU starts. @@ -204,6 +261,15 @@ 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), 'map-without-identity': ('freed_chunks_are_mapped_again', 'alias of another object'), + 'pending-reinit-stops-free': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), + 'pending-free-at-level-1': ('client_boundary_reinit_at_level_1_keeps_the_backing', 'freed at level 1'), + 'reclaim-pending-not-set': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), + 'free-on-recovery-reinit': ('recovery_reinit_frees_nothing', 'freed for a recovery'), + 'reclaim-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (64 blocks, floor 32)'), + 'drain-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (3000 blocks'), + 'reinit-at-idle-reclaim': ('same_client_render_after_1_1s', 'posted count reset before the InitBM (1500)'), + 'reinit-at-any-reclaim': ('same_client_render_after_60ms', 'posted count reset before the InitBM'), + 'freed-without-walk': ('failed_vm_walk_counts_nothing_freed', 'freed counted with the VM walk failed'), 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), 'chunks-above-16mib': ('growth_backs_before_posting', 'a growth chunk above 16 MiB'), @@ -341,6 +407,14 @@ def consts(text, names): pub static LOG: RefCell> = RefCell::new(Vec::new()); pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); pub static PARAM_BLOCKS: Cell = Cell::new(PB_BLOCKS_DEFAULT); + /// The reclaim parameters (g15_pb_reclaim, g15_pb_idle_ms, g15_pb_reclaim_blocks). + pub static PARAM_RECLAIM: Cell = Cell::new(PB_RECLAIM_DEFAULT); + pub static PARAM_IDLE_MS: Cell = Cell::new(PB_IDLE_MS_DEFAULT); + pub static PARAM_RECLAIM_BLOCKS: Cell = Cell::new(PB_RECLAIM_BLOCKS_DEFAULT); + /// The monotonic clock (engine::now_ns()). + pub static NOW: Cell = Cell::new(1_000_000_000); + /// The VM registry walk (vms.open()) fails this many times. + pub static FAIL_OPEN: Cell = Cell::new(0); /// Failures to inject: the next this many allocations or alias maps fail. pub static FAIL_ALLOC: Cell = Cell::new(0); pub static FAIL_MAP: Cell = Cell::new(0); @@ -396,6 +470,15 @@ def consts(text, names): } } pub static g15_pb_blocks: Blocks = Blocks; + pub struct Cellp(pub &'static std::thread::LocalKey>); + impl Cellp { + pub fn value(&self) -> u32 { + self.0.with(|p| p.get()) + } + } + pub static g15_pb_reclaim: Cellp = Cellp(&crate::PARAM_RECLAIM); + pub static g15_pb_idle_ms: Cellp = Cellp(&crate::PARAM_IDLE_MS); + pub static g15_pb_reclaim_blocks: Cellp = Cellp(&crate::PARAM_RECLAIM_BLOCKS); } pub mod driver { @@ -520,9 +603,31 @@ def consts(text, names): } pub struct Engine { pub flights: Vec, + /// Jobs accepted and not fully published. + pub pending: Vec, + pub frozen: bool, } pub struct Recovery { pub reinit: bool, + pub armed: Option<(usize, u64)>, + pub failed: Option, + } + impl Recovery { + pub fn failed(&self) -> Option { + self.failed + } + } + /// vm.rs's Registry: the VMs whose client handle may still be open. + pub struct Registry { + pub vms: Vec>, + } + impl Registry { + pub fn open(&self) -> Result>> { + if crate::fail(&crate::FAIL_OPEN) { + return Err(ENOMEM); + } + Ok(KVec(self.vms.clone())) + } } pub struct Receipts { pub renders: u32, @@ -562,6 +667,8 @@ def consts(text, names): pub bad: u64, pub waits: u64, pub refills: u64, + pub reclaims: u64, + pub freed: u64, } pub struct Live { pub block_ctl: FwObject, @@ -574,6 +681,8 @@ def consts(text, names): pub live: Live, pub drm: ARef, pub chunks: KVec, + pub idle: growth::IdleClock, + pub reclaim_pending: bool, } impl Pools { pub fn chunk_count(&self) -> usize { @@ -582,12 +691,6 @@ def consts(text, names): pub fn chunk_capacity(&self) -> usize { self.chunks.capacity() } - /// What a reclaim does to the backing (the PB re-initialization included). - pub fn drop_chunks(&mut self, keep: usize) { - self.chunks.truncate(keep); - self.ledger.reinit(); - self.ledger.unback_to(keep as u32); - } /// The backing objects' ids, in heap order. pub fn chunk_ids(&self) -> Vec { self.chunks.iter().map(|c| c.backing.gem.id).collect() @@ -643,6 +746,7 @@ def consts(text, names): pub init: Init, pub engine: Engine, pub recovery: Recovery, + pub vms: Registry, pub renders: Vec, pub fw: FwModel, } @@ -654,6 +758,7 @@ def consts(text, names): let vm = VmState::new(5); let mut f = Self::without_flights(); f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + f.vms.vms.push(vm.clone()); f.map_pb(&vm).unwrap(); crate::take(); f @@ -667,6 +772,8 @@ def consts(text, names): pools: Pools { ledger, exhausted_logged: false, + idle: growth::IdleClock::default(), + reclaim_pending: false, stats: PbStats::default(), drm, chunks, @@ -680,8 +787,9 @@ def consts(text, names): }, }, init: Init { control: Some(std::sync::Arc::new(Ctl { epoch: 0x1_0000_0003 })) }, - engine: Engine { flights: Vec::new() }, - recovery: Recovery { reinit: false }, + engine: Engine { flights: Vec::new(), pending: Vec::new(), frozen: false }, + recovery: Recovery { reinit: false, armed: None, failed: None }, + vms: Registry { vms: Vec::new() }, renders: vec![1], fw: FwModel { ctx: 5, @@ -832,6 +940,7 @@ def consts(text, names): /// Growth to the maximum with the firmware asking after every reply, one pass at a time. fn run_to_exhaustion(param: u32) { + defaults(); PARAM.with(|p| p.set(param)); let mut f = Firmware::new(); let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); @@ -895,6 +1004,7 @@ def consts(text, names): #[test] fn driver_values() { + defaults(); let l = Ledger::new(tuning::PB_GROW, params::pb_blocks(), tuning::PB_MAX_BLOCKS); assert!(tuning::PB_GROW); assert_eq!(crate::PB_BLOCKS_DEFAULT, 64); @@ -907,6 +1017,7 @@ def consts(text, names): #[test] fn soft_max_bounds() { + defaults(); let f = Firmware::new(); for param in [0, 1, 63, 64, 65, 1023, 1024, 1025, 3071, 3072, 3073, 4094, 4095, 4096, 8192, u32::MAX] { PARAM.with(|p| p.set(param)); @@ -1144,7 +1255,7 @@ def consts(text, names): #[test] fn growth_backs_before_posting() { - PARAM.with(|p| p.set(PARAM_DEFAULT)); + defaults(); let mut f = Firmware::new(); let start = f.pools.ledger.backed as u64; let first_chunks = f.pools.chunk_count(); @@ -1410,42 +1521,321 @@ def consts(text, names): assert_eq!(f.pools.stats.bad, 1); } - /// What a reclaim leaves behind in a VM that was not walked (a closed VM not pruned yet, or - /// one created meanwhile): its aliases of freed chunks go, and chunks backed again at the - /// same GVA are mapped from their new objects at its next publication. + /// Every PB parameter at its default. libtest gives each test its own thread, but a test + /// that sets one starts from the defaults all the same. + fn defaults() { + PARAM.with(|p| p.set(PARAM_DEFAULT)); + crate::PARAM_BLOCKS.with(|b| b.set(crate::PB_BLOCKS_DEFAULT)); + crate::PARAM_RECLAIM.with(|r| r.set(crate::PB_RECLAIM_DEFAULT)); + crate::PARAM_IDLE_MS.with(|r| r.set(crate::PB_IDLE_MS_DEFAULT)); + crate::PARAM_RECLAIM_BLOCKS.with(|r| r.set(crate::PB_RECLAIM_BLOCKS_DEFAULT)); + } + + /// A prefix of `vm` published as engine.rs does it: publish_prefix_into() first catches the + /// VM up (map_pb(), before the flight is added), and a render prefix with a + /// re-initialization pending does the InitBM (Pools::init()'s ledger reset) and clears + /// `reinit` at its publication. `render`: the prefix has renders. + fn publish(f: &mut Firmware, vm: &Arc, render: bool) { + f.map_pb(vm).unwrap(); + if render && f.recovery.reinit { + f.reset_ledger(); + f.recovery.reinit = false; + } + f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + let i = f.engine.flights.len() - 1; + if f.renders.len() <= i { + f.renders.resize(i + 1, 0); + } + f.renders[i] = render as u32; + f.fw.ctx = vm.slot; + f.note_idle(); + } + + /// What a reclaim leaves behind in a VM that was not walked (a closed VM no longer in the + /// registry, whose accepted jobs may still run): its aliases of freed chunks go, and chunks + /// backed again at the same GVA are mapped from their new objects at its next publication. + /// The reclaims are the actual idle reclaim; the posted count stays until the InitBM. #[test] fn freed_chunks_are_mapped_again() { + defaults(); let mut f = Firmware::new(); let a = f.engine.flights[0].job.vm.clone(); - grow_to(&mut f, 300); - let chunks = f.pools.chunk_count(); + grow_to(&mut f, 1100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); assert!(chunks > 2); - // The reclaim frees the chunks above the kept two; A is not walked. + // A's work retires and A leaves the registry; the idle reclaim frees the chunks above + // the kept two without walking A. f.engine.flights.clear(); - f.pools.drop_chunks(2); + f.vms.vms.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); take(); - // B runs and grows the backing again: new objects at the old GVAs. + // B runs: its first render re-initializes; it grows the backing again, new objects at + // the old GVAs. let b = VmState::new(6); - f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); - f.fw.ctx = 6; - f.map_pb(&b).unwrap(); - grow_to(&mut f, 300); + f.vms.vms.push(b.clone()); + publish(&mut f, &b, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 1100); assert_eq!(f.pools.chunk_count(), chunks); // A runs again: its stale aliases are replaced by the new objects'. f.engine.flights.clear(); - f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); take(); - f.map_pb(&a).unwrap(); + publish(&mut f, &a, true); let log = take(); assert_cursor(&f, &a, chunks); assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), chunks - 2); - // With fewer chunks than A maps, the ones past the backing go too. - f.pools.drop_chunks(2); + // Another idle reclaim walks B, not A: with fewer chunks than A maps, A's aliases past + // the backing go at its next catch-up. + f.engine.flights.clear(); + let t1 = now(); + check_at(&mut f, t1); + check_at(&mut f, t1 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + assert_cursor(&f, &b, 2); f.map_pb(&a).unwrap(); assert_cursor(&f, &a, 2); } + /// A reclaim whose walk of the VM registry fails: the chunks leave the backing, but a VM's + /// aliases still hold their pages, so nothing is counted freed; the VM drops its stale + /// aliases at its next publication, and only then are the pages released. + #[test] + fn failed_vm_walk_counts_nothing_freed() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1500); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + take(); + crate::FAIL_OPEN.with(|c| c.set(1)); + check_at(&mut f, t0 + 1_000_000_000); + let log = take(); + assert_eq!(f.pools.chunk_count(), 2); + assert_eq!(f.pools.stats.freed, 0, "freed counted with the VM walk failed"); + assert!(log.iter().any(|e| matches!(e, Ev::Log("warn", m) if m.contains("could not walk the VMs")))); + assert!(log.iter().any(|e| matches!(e, Ev::Log("info", m) if m.contains(" 0 chunks freed")))); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), 0, "pages released while aliases hold them"); + assert_eq!(a.heap.list.lock().len(), chunks); + // A's next publication drops the stale aliases: the pages go. + publish(&mut f, &a, true); + let log = take(); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + assert_cursor(&f, &a, 2); + } + + /// One client idles 60 ms after a heavy scene (more than half the maximum posted) and + /// renders again: the client-boundary re-initialization was set at 50 ms with the backing + /// kept, the posted count stays until the render's InitBM, which posts the initial blocks + /// again, and the scene grows again from the kept backing with nothing allocated or mapped. + #[test] + fn same_client_render_after_60ms() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 60_000_000); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!( + (f.pools.ledger.posted, f.pools.chunk_count()), + (posted, chunks), + "posted count reset before the InitBM" + ); + take(); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + assert!(!f.recovery.reinit && !f.pools.reclaim_pending); + grow_to(&mut f, 2100); + let log = take(); + assert_eq!(count(&log, is_backing), 0, "backing changed: {:?}", log.iter().filter(|e| is_backing(e)).collect::>()); + assert_posted_backed(&log, 5, f.pools.ledger.backed as u64); + } + + /// One client idles 1.1 s and renders again, after a scene above the reclaim floor (the + /// idle reclaim at 1 s) or above half the maximum (the client boundary at 50 ms, then the + /// free at 1 s): the backing above the kept chunks was freed and unmapped from the client's + /// VM, the posted count stays until the render's InitBM, and the scene grows again with + /// every chunk backed and mapped before a record or producer names it. + #[test] + fn same_client_render_after_1_1s() { + for target in [1500, 2100] { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, target); + let posted = f.pools.ledger.posted; + f.engine.flights.clear(); + let t0 = now(); + for dt in [0, 50_000_000, 1_000_000_000, 1_100_000_000] { + check_at(&mut f, t0 + dt); + } + assert_eq!(f.pools.chunk_count(), 2, "not freed ({target})"); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM ({target})"); + assert_cursor(&f, &a, 2); + take(); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, target); + let log = take(); + assert_posted_backed(&log, 5, 2 * CHUNK_BLOCKS as u64); + assert_cursor(&f, &a, f.pools.chunk_count()); + } + } + + /// A compute-only submission between a level-2 reclaim and the next render's InitBM: its + /// publication maps nothing, its passes back and post nothing (the posted count is still + /// the old one, above the kept backing), and a growth request naming its context is left + /// unanswered; once it retires, the render after it re-initializes as usual. + #[test] + fn compute_between_reclaim_and_initbm() { + defaults(); + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1500); + let posted = f.pools.ledger.posted; + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 1_000_000_000); + assert_eq!(f.pools.chunk_count(), 2); + let bad = f.pools.stats.bad; + take(); + publish(&mut f, &a, false); + f.queue(); + for _ in 0..4 { + f.pass(); + } + let log = take(); + let touched: Vec<&Ev> = log + .iter() + .filter(|e| is_backing(e) || matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_))) + .collect(); + assert!(touched.is_empty(), "compute-only flight backed or posted: {touched:?}"); + assert!(f.recovery.reinit && f.pools.ledger.posted == posted); + assert_eq!(f.pools.stats.bad, bad + 1); + // The compute retires; the check finds nothing more to do; the next render re-inits. + f.engine.flights.clear(); + check_at(&mut f, t0 + 2_000_000_000); + assert_eq!(count(&take(), is_backing), 0); + publish(&mut f, &a, true); + assert_eq!(f.pools.ledger.posted, 64); + grow_to(&mut f, 300); + assert_posted_backed(&take(), 5, 2 * CHUNK_BLOCKS as u64); + } + + fn now() -> u64 { + crate::NOW.with(|n| n.get()) + } + + /// The idle clock at `t` and the reclaim check at that time, as a job arrival or an idle + /// poll pass runs it. + fn check_at(f: &mut Firmware, t: u64) { + crate::NOW.with(|n| n.set(t)); + f.note_idle(); + f.reclaim_check(); + } + + /// A heavy client (more than half the maximum posted) exits and nothing renders after it: + /// the 50 ms client-boundary re-initialization keeps the backing, and a check after the + /// long idle frees the backing above the kept chunks. The posted count stays until the next + /// render's InitBM. + #[test] + fn client_boundary_reinit_frees_without_a_render() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(posted > 2047 && chunks > 2); + // The pass that retires the last flight notes the idle. + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + take(); + check_at(&mut f, t0 + 50_000_000); + assert!(f.recovery.reinit && f.pools.chunk_count() == chunks, "client-boundary re-initialization"); + check_at(&mut f, t0 + 999_999_999); + assert_eq!(f.pools.chunk_count(), chunks, "freed before the long idle"); + check_at(&mut f, t0 + 1_000_000_000); + let log = take(); + assert_eq!(f.pools.chunk_count(), 2, "kept backing not freed: {} chunks", f.pools.chunk_count()); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!(f.pools.stats.reclaims, 1); + assert_eq!(f.pools.ledger.backed, 2 * CHUNK_BLOCKS); + assert_eq!(f.pools.stats.freed, (chunks - 2) as u64); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + // Later checks find nothing more to do. + check_at(&mut f, t0 + 100_000_000_000); + assert!(take().is_empty()); + } + + /// With g15_pb_reclaim 1 the client-boundary re-initialization keeps the backing for good. + #[test] + fn client_boundary_reinit_at_level_1_keeps_the_backing() { + defaults(); + crate::PARAM_RECLAIM.with(|r| r.set(1)); + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 50_000_000); + assert!(f.recovery.reinit); + check_at(&mut f, t0 + 5_000_000_000); + assert_eq!(f.pools.chunk_count(), chunks, "freed at level 1"); + assert_eq!(f.pools.stats.freed, 0); + } + + /// Nothing grown since the initialization: an initial posting above half the maximum + /// (g15_pb_blocks 3000), or a floor below it (g15_pb_reclaim_blocks 32): no + /// re-initialization at any idle. + #[test] + fn nothing_grown_reinitializes_nothing() { + for (blocks, floor) in [(64, 32), (3000, 1024)] { + defaults(); + crate::PARAM_BLOCKS.with(|b| b.set(blocks)); + crate::PARAM_RECLAIM_BLOCKS.with(|b| b.set(floor)); + let mut f = Firmware::new(); + assert_eq!(f.pools.ledger.posted, blocks); + f.engine.flights.clear(); + let t0 = now(); + for dt in [0, 50_000_000, 1_000_000_000, 5_000_000_000] { + check_at(&mut f, t0 + dt); + } + assert!( + !f.recovery.reinit && f.pools.stats.reclaims == 0, + "re-initialization with nothing grown ({blocks} blocks, floor {floor})" + ); + } + } + + /// A recovery's re-initialization (not a reclaim's) is left alone: nothing freed. + #[test] + fn recovery_reinit_frees_nothing() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + f.recovery.reinit = true; + let t0 = now(); + check_at(&mut f, t0); + check_at(&mut f, t0 + 5_000_000_000); + assert_eq!(f.pools.chunk_count(), chunks, "freed for a recovery's re-initialization"); + assert_eq!(f.pools.stats.reclaims, 0); + } + #[test] fn catch_up_refuses_flights_of_another_vm() { let mut f = Firmware::new(); @@ -1546,9 +1936,13 @@ def run(mutation): bodies = [item(pools, s) for s in (' pub(crate) fn answer_growth(&mut self) -> Result {', ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', ' pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result {', - ' pub(crate) fn refill_backing(&mut self) {')] + ' pub(crate) fn refill_backing(&mut self) {', + ' pub(crate) fn note_idle(&mut self) {', + ' pub(crate) fn reclaim_check(&mut self) {', + ' fn free_backing(&mut self) -> usize {')] soft_max = item(pools, ' pub(crate) fn soft_max(&self) -> u32 {') pools_methods = (item(pools, ' fn map_heap(&self, vm: &VmState) -> Result {') + '\n' + + item(pools, ' fn free_chunks(&mut self, keep: usize) -> usize {') + '\n' + item(pools, ' fn back_chunk(&mut self) -> Result> {')) chunk = item(pools, 'struct PbChunk {') + '\n' + item(pools, 'impl PbChunk {') chunk_geometry = (re.search(r'^pub\(crate\) const CHUNK_BYTES: usize = [^;]+;', pools, re.M).group(0) + '\n' + @@ -1582,8 +1976,16 @@ def run(mutation): if init.count(line) != 1: raise SystemExit('Pools::init() does not hold: ' + line) reset = '\n'.join('p.' + line[len('self.'):] for line in LEDGER_RESET) - pb_max_blocks = (item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> u32 {') + '\n' + - item(src[PARAMS], 'pub(crate) fn pb_blocks() -> u32 {')) + pb_max_blocks = '\n'.join(item(src[PARAMS], f'pub(crate) fn {name}() -> ') + for name in ('pb_max_blocks', 'pb_blocks', 'pb_reclaim', 'pb_idle_ns', + 'pb_reclaim_blocks')) + reclaim_defaults = {} + for param, const in (('g15_pb_reclaim', 'PB_RECLAIM_DEFAULT'), ('g15_pb_idle_ms', 'PB_IDLE_MS_DEFAULT'), + ('g15_pb_reclaim_blocks', 'PB_RECLAIM_BLOCKS_DEFAULT')): + m = re.search(rf'{param}: u32 \{{\s*default: (\d+),', src[CORE]) + if not m: + raise SystemExit(f'{param} default not found') + reclaim_defaults[const] = int(m.group(1)) m = re.search(r'g15_pb_max_blocks: u32 \{\s*default: (\d+),', src[CORE]) if not m: raise SystemExit('g15_pb_max_blocks default not found') @@ -1608,6 +2010,7 @@ def run(mutation): (dir / Path(path).name).write_text(src[path]) code = (PREFIX + f'pub const PARAM_DEFAULT: u32 = {param_default};\n' + f'pub const PB_BLOCKS_DEFAULT: u32 = {blocks_default};\n' + + ''.join(f'pub const {k}: u32 = {v};\n' for k, v in reclaim_defaults.items()) + 'pub mod g15 {\n' ' pub mod fw {\n' f' #[path = "{dir / "channels.rs"}"] pub mod channels;\n' @@ -1624,6 +2027,7 @@ def run(mutation): ' }\n' ' pub mod pgtable {\n' + pg_consts + '\n' + prot + '\n }\n' ' pub mod boot {\n pub fn eproto() -> crate::Error { crate::EPROTO }\n }\n' + ' pub mod engine {\n pub fn now_ns() -> u64 { crate::NOW.with(|n| n.get()) }\n }\n' ' /// The checked backing allocation, failing where a test injects it.\n' ' pub mod object {\n' ' use crate::prelude::*;\n' From ddd0598a4f0605f11b7bea48d392cb8c38f99bdd Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:13:57 +0530 Subject: [PATCH 06/46] drm/asahi: g15: m3: bound the CDM stream walk by bytes, not by 8192 records The CDM gate walks a compute stream from its base through its links and refuses it after 8192 records. A legitimate stream can be longer. On the J516C, the three dEQP-VK multi_draw.mosaic cases with max_draws and tessellation (1024 tessellated draws, several launches and barriers each) were refused: "rejecting CDM record budget record", then EINVAL from the submit. The gate is shared by the T6030 and the T6031. It already admits at most 64 links, and Mesa's encoder links to a new 64 KiB block when one fills, so a stream that fits the link budget is at most 65 blocks. Bound the walk by the bytes of its records instead: at most (64 + 1) * 64 KiB (4160 KiB) of stream, whatever the mix of 4-byte barriers and 24- to 40-byte launches. A stream without TERMINATE still ends at that bound or at the link budget, so the walk stays bounded: at most 1064960 record reads of 4 bytes. A walk does not check again the last page it found good, for a program or for an indirect launch's 12 argument bytes, and it makes at most 16384 page checks; one more is refused as "CDM page check budget". The walk holds no device lock: the queue's mutex covers the SUBMIT, and the VM's lock is taken per check (about 4 us on the J516C) and per 256-byte read (about 5 us). Without the cache and the bound, a stream of 24-byte indirect launches with distinct argument pages made one check per launch: about 0.8 s per compute command and about 51 s per 64-command SUBMIT. Now it is at most about 149 ms per command and about 9.5 s per SUBMIT. Mesa's indirect arguments move to a new page every few dispatches: 40000 dispatches made 41410 checks in 224 ms before and 6848 in 94 ms after, and the largest stream the link budget admits (47000 dispatches) makes 8047. The 608 dEQP-VK multi_draw.mosaic max_draws with_tess cases pass. Host tests (asahi-g15 selftest): 10240 barriers and a terminator are admitted; a terminator in the last word of the budget is admitted and one past it refused; a stream of the budget without a terminator is refused; 64 links are admitted and 65 refused; 1000 launches in one argument page make 2 page checks; a program and 16383 argument pages are admitted and one more page is refused without its check; 170000 launches over 16 argument pages are admitted with 17 checks. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit ae93621adf50dfc39c549b25642e22dfc95cf7ac) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/gate.rs | 92 +++++++++++++++---- .../selftests/drivers/gpu/asahi-g15/run.py | 92 ++++++++++++++++++- 2 files changed, 164 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/gate.rs b/drivers/gpu/drm/asahi/g15/gate.rs index f41b2e3f75f57..a6fb20bf156b5 100644 --- a/drivers/gpu/drm/asahi/g15/gate.rs +++ b/drivers/gpu/drm/asahi/g15/gate.rs @@ -3,11 +3,13 @@ //! Stream admission gates (safety nets for userspace encoders). //! //! VDM: the root stream's first word is 0x21000960. CDM walk from `cdm_ctrl_stream_base` through -//! links until TERMINATE (0x40000000), at most 8192 records and 64 links (a longer or cyclic -//! stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; +//! links until TERMINATE (0x40000000), at most 64 links and [`CDM_MAX_BYTES`] of records (a +//! longer or cyclic stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; //! link 0x2000XXXX; direct launch (40 B, the G15 layout), indirect launches 36 B //! (`w0 & 0xff7fffff == 0x08080000`) and 24 B (0x10080000); every program VA inside the queue's -//! 4 GiB USC window, 64-aligned, first 64 bytes readable. Client memory is read through the VM's +//! 4 GiB USC window, 64-aligned, first 64 bytes readable; an indirect launch's 12 argument bytes +//! readable. A program or argument page found good is not checked again in the same walk, and a +//! walk checks at most [`CDM_MAX_PAGE_CHECKS`] pages. Client memory is read through the VM's //! page tables with a point-of-coherency clean+invalidate of the lines ([`mmu::Vm::read_user`]: //! the GPU may have written it). `g15_stream_gate=0` disables both gates (encoder bring-up only). @@ -24,8 +26,19 @@ const CDM_LINK: u32 = 0x20000000; const CDM_DIRECT: u32 = 0x00080000; const CDM_INDIRECT_SIZES: u32 = 0x08080000; const CDM_INDIRECT: u32 = 0x10080000; -const CDM_MAX_RECORDS: u32 = 8192; const CDM_MAX_LINKS: u32 = 64; +/// The stream block the encoder links to when one fills (Mesa's 64 KiB control-stream blocks). +const CDM_BLOCK_BYTES: u64 = 0x10000; +/// Stream bytes walked at most: the root block and one block per admitted link. A stream of +/// that size holds every record the encoder can write before the link budget (a record count +/// does not: the 1024 tessellated draws of one multi-draw write more than 8192). +const CDM_MAX_BYTES: u64 = (CDM_MAX_LINKS as u64 + 1) * CDM_BLOCK_BYTES; +/// Page checks (a program's or an indirect launch's arguments, each under the VM lock) a walk +/// makes at most. The last page found good is not checked again, but the encoder's indirect +/// arguments move to a new page every few launches: a stream at the link budget makes about +/// 8100. Twice that leaves room; without a bound, a stream of distinct pages would make one per +/// 24-byte launch, about 177000 in the byte budget. +const CDM_MAX_PAGE_CHECKS: u32 = 16384; /// Rejections logged per boot. const MAX_REPORTS: u32 = 32; @@ -86,19 +99,52 @@ impl Window { } } +/// The pages a walk has found good and the page checks it has made. +struct Checked { + program: Option, + args: Option, + count: u32, +} + +/// One page check of `[start, end)` under the walk's page budget: ENOSPC once the budget is +/// spent. +fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result { + if *count >= CDM_MAX_PAGE_CHECKS { + return Err(ENOSPC); + } + *count += 1; + cover(vm, start, end, true, false) +} + /// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable. -/// `checked` caches the last proven program page. -fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Option) -> Result { +/// The last proven program page is not checked again. +fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Result { if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 { return Err(EINVAL); } // 64-aligned: the block never crosses its page. let page = address & !(mmu::UAT_PGMSK as u64); - if *checked == Some(page) { + if checked.program == Some(page) { return Ok(()); } - cover(vm, address, address + 64, true, false)?; - *checked = Some(page); + check(vm, address, address + 64, &mut checked.count)?; + checked.program = Some(page); + Ok(()) +} + +/// An indirect launch's 12 argument bytes: readable. The last proven argument page is not +/// checked again; arguments that cross a page are checked each time. +fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result { + let end = address.checked_add(12).ok_or(EINVAL)?; + let page = address & !(mmu::UAT_PGMSK as u64); + let within = (end - 1) & !(mmu::UAT_PGMSK as u64) == page; + if within && checked.args == Some(page) { + return Ok(()); + } + check(vm, address, end, &mut checked.count)?; + if within { + checked.args = Some(page); + } Ok(()) } @@ -128,15 +174,19 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { len: 0, bytes: [0; 256], }; - let mut checked = None; + let mut checked = Checked { + program: None, + args: None, + count: 0, + }; let mut va = base; let mut links = 0; - let mut records = 0; - while records < CDM_MAX_RECORDS { - records += 1; + let mut walked = 0; + while walked < CDM_MAX_BYTES { let w = window.words(vm, va, 1)?; if CDM_BARRIERS.contains(&w[0]) { va += 4; + walked += 4; continue; } if w[0] == CDM_TERMINATE { @@ -152,6 +202,7 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { if va & 3 != 0 { return Err(reject("CDM link", va, &w)); } + walked += 8; continue; } if w[0] == CDM_DIRECT { @@ -162,9 +213,13 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { } match program(vm, usc, address, &mut checked) { Err(error) if error == EINVAL => return Err(reject("CDM program", va, &w)), + Err(error) if error == ENOSPC => { + return Err(reject("CDM page check budget", va, &w)) + } result => result?, } va += 40; + walked += 40; continue; } let kind = w[0] & 0xff7fffff; @@ -182,15 +237,20 @@ pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { return Err(reject("CDM indirect launch", va, &w)); } let result = program(vm, usc, address, &mut checked) - .and_then(|()| cover(vm, indirect, indirect + 12, true, false)); + .and_then(|()| arguments(vm, indirect, &mut checked)); match result { Err(error) if error == EINVAL => return Err(reject("CDM indirect", va, &w)), + Err(error) if error == ENOSPC => { + return Err(reject("CDM page check budget", va, &w)) + } result => result?, } - va += if sizes { 36 } else { 24 }; + let size = if sizes { 36 } else { 24 }; + va += size; + walked += size; continue; } return Err(reject("CDM", va, &w)); } - Err(reject("CDM record budget", va, &[])) + Err(reject("CDM byte budget", va, &[])) } diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 35c96a7f36176..861290ef24098 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -102,6 +102,7 @@ pub type Result = std::result::Result; pub const EINVAL: Error = -22; pub const EFAULT: Error = -14; + pub const ENOSPC: Error = -28; pub use crate::{pr_warn, static_assert}; } #[macro_export] @@ -124,8 +125,19 @@ } mod command { use crate::{mmu, prelude::*}; + use std::cell::Cell; pub const USC_WINDOW: u64 = 1 << 32; + /// Addresses from here on are mapped and readable without backing in the mock VM. + pub const SYNTHETIC: u64 = 1 << 40; + thread_local! { + /// cover() calls: the page checks the walk made. + pub static COVERS: Cell = Cell::new(0); + } pub fn cover(vm: &mmu::Vm, start: u64, end: u64, _: bool, _: bool) -> Result { + COVERS.with(|c| c.set(c.get() + 1)); + if start >= SYNTHETIC { + return Ok(()); + } let mut bytes = vec![0; usize::try_from(end - start).map_err(|_| EINVAL)?]; vm.read_user(start, &mut bytes) } @@ -151,16 +163,88 @@ fn cyclic_links_are_rejected() { assert_eq!(gate::cdm(&vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); } + /// Words of 4 bytes that fill the byte budget: the root block and one per link. + const BUDGET_WORDS: usize = 65 * 0x10000 / 4; #[test] - fn record_budget_requires_a_terminator() { - assert_eq!(gate::cdm(&vm(&vec![0x60000160; 8192]), 0, 0), Err(EINVAL)); + fn byte_budget_requires_a_terminator() { + assert_eq!(gate::cdm(&vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); } #[test] fn final_record_may_terminate() { - let mut words = vec![0x60000160; 8192]; - words[8191] = 0x40000000; + let mut words = vec![0x60000160; BUDGET_WORDS + 1]; + words[BUDGET_WORDS - 1] = 0x40000000; + assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + words[BUDGET_WORDS - 1] = 0x60000160; + words[BUDGET_WORDS] = 0x40000000; + assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + } + #[test] + fn more_than_8192_records_are_admitted() { + // 1024 draws of ten records each, as a tessellated multi-draw writes them. + let mut words = vec![0x60000160; 10240]; + words.push(0x40000000); + assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + } + #[test] + fn link_budget_still_bounds_the_walk() { + // 65 links, each to the next: the 65th is refused. + let mut words = Vec::new(); + for i in 0..65u32 { + words.extend([0x20000000, (i + 1) * 8]); + } + words.push(0x40000000); + assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + let mut words = Vec::new(); + for i in 0..64u32 { + words.extend([0x20000000, (i + 1) * 8]); + } + words.push(0x40000000); assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); } + /// The USC window of these cases and its first program, both in the mock's mapped range. + const USC: u64 = crate::command::SYNTHETIC; + /// A 24-byte indirect launch of the window's first program with its arguments at `args`. + fn indirect(args: u64) -> [u32; 6] { + [0x10080000, ((USC >> 38) as u32) << 22, (USC >> 6) as u32, + 0x40000000 | ((USC >> 40) & 0xff) as u32, (args >> 32) as u32 & 0xffff, args as u32] + } + /// `n` indirect launches, launch `i` with its arguments at `args(i)`, then TERMINATE. + fn launches(n: usize, args: impl Fn(usize) -> u64) -> Vec { + let mut words: Vec = (0..n).flat_map(|i| indirect(args(i))).collect(); + words.push(0x40000000); + words + } + fn covers(words: &[u32]) -> (Result, u32) { + crate::command::COVERS.with(|c| c.set(0)); + let r = gate::cdm(&vm(words), USC, 0); + (r, crate::command::COVERS.with(|c| c.get())) + } + const PAGE: u64 = mmu::UAT_PGSZ as u64; + #[test] + fn repeated_argument_pages_are_checked_once() { + // 1000 launches with their arguments in one page: the program page and that page. + let base = 2 * USC; + assert_eq!(covers(&launches(1000, |i| base + (i as u64 % 100) * 12)), (Ok(()), 2)); + // Alternating between two pages: one check per change of page. + assert_eq!(covers(&launches(10, |i| base + (i as u64 % 2) * PAGE)), (Ok(()), 11)); + // Arguments that cross a page are checked each time. + assert_eq!(covers(&launches(3, |_| base + PAGE - 8)), (Ok(()), 4)); + } + #[test] + fn page_check_budget_bounds_the_walk() { + let base = 2 * USC; + // The program page and 16383 argument pages: 16384 checks, admitted. + assert_eq!(covers(&launches(16383, |i| base + i as u64 * PAGE)), (Ok(()), 16384)); + // One more page: refused at the 16385th check, which is not made. + assert_eq!(covers(&launches(16384, |i| base + i as u64 * PAGE)), (Err(EINVAL), 16384)); + } + #[test] + fn a_large_stream_of_reused_pages_is_admitted() { + // 170000 launches (4 MB, inside the byte budget) over 16 argument pages. + let base = 2 * USC; + let words = launches(170000, |i| base + (i as u64 / 10625) * PAGE + (i as u64 % 1000) * 12); + assert_eq!(covers(&words), (Ok(()), 17)); + } #[test] fn malformed_and_unreadable_streams_are_rejected() { assert_eq!(gate::cdm(&vm(&[0xffffffff]), 0, 0), Err(EINVAL)); From 6ce5b84d53de590860bc5e1487168bcc1da6a2f9 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:14:08 +0530 Subject: [PATCH 07/46] drm/asahi: g15: select the Rust firmware loader abstractions The G15 backend loads the T6031 board images with kernel::firmware (g15/board/blob.rs), which exists only with CONFIG_RUST_FW_LOADER_ABSTRACTIONS. DRM_ASAHI_G15 does not select it: the symbol comes only from APPLE_AOP, APPLE_SEP or nova-core, so a configuration without them fails to build the driver. Select RUST_FW_LOADER_ABSTRACTIONS from DRM_ASAHI_G15. It depends only on RUST, which DRM_ASAHI already requires, and selects FW_LOADER. Built with APPLE_AOP and APPLE_SEP off: the option resolves to y and drivers/gpu/drm/asahi builds. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit c562b82dd6654017e6404d83b00d809ae298cc04) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 647f8f37507d4..2e8ca81a64505 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -60,6 +60,7 @@ config DRM_ASAHI_G15 depends on DRM_APPLE=y || (DRM_APPLE=m && DRM_ASAHI=m) select APPLE_T6030_GPU_PMP_LINK select CRYPTO_LIB_SHA256 + select RUST_FW_LOADER_ABSTRACTIONS default n help Enable the native G15 backend for explicitly admitted M3 Pro/Max From 48142e6e63a923124dc8c9fb7a155219bfdfa87e Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:14:52 +0530 Subject: [PATCH 08/46] drm/asahi: g15: m3: run the PB reclaim check after the idle time without new work The reclaim check runs in poll passes and when a job arrives, but an idle engine runs no poll pass: the pass that retires the last flight is not re-armed. On the J516C, a client-boundary re-initialization kept 23 chunks (368 MiB) for a client that then posted 264 blocks. With no new job they were still held 15 s later, and only the next job's arrival freed 21 of them. When a poll pass ends with the engine idle, compute how much longer the engine must stay idle before the reclaim check would act (Ledger::reclaim_delay(): the idle reclaim at g15_pb_idle_ms, the free of kept backing at level 2, or the client-boundary re-initialization). Then queue the poll work for that delay. The pass it runs takes the device lock and runs the idle path: note_idle(), then reclaim_check(). Nothing is armed while busy, while a recovery or its re-initialization is pending, or when nothing can become due. While a reclaim's re-initialization is pending (a client-boundary one keeps the backing), the poll is armed for the rest of g15_pb_idle_ms at level 2 (Ledger::pending_free_delay()), not for the expired 50 ms, so the check frees the kept backing of a heavy client that exits with no render after it. A job's arrival re-arms the poll work for its publication, which replaces the expiry. The delay mirrors the checks, also in requiring growth since the initialization. This applies to the T6030 and the T6031, which share this code; it is untested on the T6030 (g15_pb_reclaim=0 turns the reclaim off, and g15_pb_blocks=1024 restores the old start). Host test (pools/growth.rs): over levels, floors, idle times, posted counts and backing, the delay is the first idle time at which reclaim_due(), free_due() or drain_reinit_due() holds. None of them holds one nanosecond before it, and none ever holds when there is no delay. The J516C case gives g15_pb_idle_ms at level 2 and nothing at levels 0 and 1. The growth harness also runs reclaim_delay_ns() and drives the idle poll work as poll.rs arms it: a heavy client that exits with nothing rendered after it wakes the poll at 54 ms (the re-initialization) and at 1004 ms (the free), and nothing is armed after; an idle reclaim arms nothing once its re-initialization is pending; nothing is armed at level 1, for a recovery's re-initialization, or when nothing grew. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit aa142eed3a7cea0d48f5b98a8f7e1f7518b5aaf1) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/poll.rs | 12 +- drivers/gpu/drm/asahi/g15/pools.rs | 31 ++++ drivers/gpu/drm/asahi/g15/pools/growth.rs | 133 ++++++++++++++++- tools/asahi/g15-pb-growth-tests.py | 165 ++++++++++++++++++++-- 4 files changed, 324 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index e719e8e7ec3c2..4e27f8ff60e99 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -23,7 +23,7 @@ use kernel::{ hrtimer::{ HrTimerCallback, HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode, }, - Delta, Jiffies, Monotonic, + msecs_to_jiffies, Delta, Jiffies, Monotonic, }, workqueue::RawWorkItem, }; @@ -85,7 +85,15 @@ impl Shared { let evented = self.evented.swap(false, Ordering::AcqRel); match firmware.poll(evented, &mut done) { Ok(true) => self.rearm(firmware.engine.chan_wait), - Ok(false) => {} + // Idle: nothing polls until new work arrives, so a PB reclaim that comes + // due with more idle time gets one more pass then. A job's arrival brings + // the poll work forward (`rearm`), which replaces this expiry. + Ok(false) => { + if let Some(ns) = firmware.reclaim_delay_ns() { + let ms = ns.div_ceil(1_000_000).min(u32::MAX as u64) as u32; + self.mod_poll(msecs_to_jiffies(ms) + 1); + } + } Err(error) => self.fail_stop(error), } } diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index f03b196760b6a..3364b136b9287 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -871,6 +871,37 @@ impl Firmware { Ok(()) } + /// How long the idle engine must stay idle before [`Self::reclaim_check`] acts, if it can + /// act without new work ([`growth::Ledger::reclaim_delay`], or with a reclaim's + /// re-initialization pending [`growth::Ledger::pending_free_delay`]); `None` while busy, + /// while a recovery or its re-initialization is pending, or when nothing would become due. + /// An idle engine runs no poll pass by itself, so the poll work is armed for this delay. + pub(crate) fn reclaim_delay_ns(&self) -> Option { + if !self.engine.flights.is_empty() + || !self.engine.pending.is_empty() + || (self.recovery.reinit && !self.pools.reclaim_pending) + || self.recovery.armed.is_some() + || self.engine.frozen + || self.recovery.failed().is_some() + { + return None; + } + let idle_ns = self.pools.idle.idle_ns(super::engine::now_ns()); + if self.recovery.reinit { + return self.pools.ledger.pending_free_delay( + super::params::pb_reclaim(), + idle_ns, + super::params::pb_idle_ns(), + ); + } + self.pools.ledger.reclaim_delay( + super::params::pb_reclaim(), + super::params::pb_reclaim_blocks(), + idle_ns, + super::params::pb_idle_ns(), + ) + } + /// Processes one growth request: `Ok(false)` when it waits for backing. fn answer_request(&mut self, request: &[u32; 14]) -> Result { let ctl = self.init.control.clone().ok_or(ENODEV)?; diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index c98aef060f94e..fef40a6d3a73e 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -179,6 +179,22 @@ impl Ledger { idle_ns >= min_idle_ns && self.backed > self.keep_chunks() * CHUNK_BLOCKS } + /// How much longer the engine must stay idle, from `idle_ns`, before + /// [`Self::pending_free_due`] with `level`; `None` when it cannot become due (level below + /// 2, or nothing above the kept chunks). 0 when it is due now. + pub(crate) const fn pending_free_delay( + &self, + level: u32, + idle_ns: u64, + min_idle_ns: u64, + ) -> Option { + if level >= 2 && self.backed > self.keep_chunks() * CHUNK_BLOCKS { + Some(min_idle_ns.saturating_sub(idle_ns)) + } else { + None + } + } + /// Whether a short idle re-initializes the heap: more than half the maximum posted and /// idle for [`DRAIN_IDLE_NS`]. Back-to-back heavy scenes (a new client every few hundred /// ms) never reach `g15_pb_idle_ms`, and posted blocks never come back while a PB lives, so @@ -189,6 +205,36 @@ impl Ledger { self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS } + /// How much longer the engine must stay idle, from `idle_ns`, before a reclaim with + /// `level` is due ([`Self::reclaim_due`] with `floor` and `min_idle_ns`, [`Self::free_due`] + /// at level 2, or [`Self::drain_reinit_due`]); `None` when none can become due while the + /// ledger stays as it is. 0 when one is due now. + pub(crate) const fn reclaim_delay( + &self, + level: u32, + floor: u32, + idle_ns: u64, + min_idle_ns: u64, + ) -> Option { + if level == 0 { + return None; + } + let grown = self.posted > self.initial; + let mut due = u64::MAX; + if grown && self.posted > self.max / 2 { + due = DRAIN_IDLE_NS; + } + let free = level >= 2 && self.backed > Self::chunks_for(self.posted) * CHUNK_BLOCKS; + if ((grown && self.posted > floor) || free) && min_idle_ns < due { + due = min_idle_ns; + } + if due == u64::MAX { + None + } else { + Some(due.saturating_sub(idle_ns)) + } + } + /// Whether the refusals at the maximum are used up: further requests go unanswered. pub(crate) const fn exhausted(&self) -> bool { self.refusals >= MAX_REFUSALS @@ -722,8 +768,84 @@ mod tests { assert!(l.reclaim_due(floor, clock.idle_ns(t), min_idle)); } + /// The delay names exactly the idle time at which the reclaim check first acts: nothing is + /// due one nanosecond before it, something is due at it, and nothing ever becomes due + /// when it is `None`. + #[test] + fn reclaim_delay_matches_the_checks() { + let s = 1_000_000_000; + let due = |l: &Ledger, level: u32, floor: u32, idle: u64, min_idle: u64| { + level != 0 + && (l.reclaim_due(floor, idle, min_idle) + || (level >= 2 && l.free_due(idle, min_idle)) + || l.drain_reinit_due(idle)) + }; + for level in 0..=2 { + for floor in [0, 64, 1024, 4095] { + for min_idle in [DRAIN_IDLE_NS / 2, 100_000_000, s] { + for posted in [64, 200, 1024, 1025, 2047, 2048, 2900, 4095] { + for chunks in [2, 3, 9, 23, MAX_CHUNKS] { + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(chunks); + l.posted = posted; + match l.reclaim_delay(level, floor, 0, min_idle) { + None => { + for idle in [0, DRAIN_IDLE_NS, min_idle, 100 * s] { + assert!(!due(&l, level, floor, idle, min_idle)); + } + } + Some(d) => { + assert!(due(&l, level, floor, d, min_idle)); + assert!(d == 0 || !due(&l, level, floor, d - 1, min_idle)); + // Counted from an idle already begun. + assert_eq!( + l.reclaim_delay(level, floor, d / 2, min_idle), + Some(d - d / 2) + ); + assert_eq!( + l.reclaim_delay(level, floor, d + s, min_idle), + Some(0) + ); + } + } + } + } + } + } + } + // The case of the J516C run: a client-boundary re-initialization left 23 chunks for + // 264 posted blocks; the free is due after g15_pb_idle_ms of idle. + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(23); + l.posted = 264; + assert_eq!(l.reclaim_delay(2, 1024, 0, s), Some(s)); + assert_eq!(l.reclaim_delay(1, 1024, 0, s), None); + assert_eq!(l.reclaim_delay(0, 1024, 0, s), None); + // A desktop at the initial posting with the kept chunks: nothing to wait for. + l.unback_to(l.keep_chunks()); + l.posted = 64; + assert_eq!(l.reclaim_delay(2, 1024, 0, s), None); + // Nothing grown, with an initial posting above half the maximum or above the floor: + // nothing armed. + let mut l = Ledger::new(true, 3000, 4095); + l.unback_to(Ledger::chunks_for(3000)); + assert_eq!( + l.reclaim_delay(2, 1024, 0, s), + None, + "armed with nothing grown" + ); + let mut l = Ledger::new(true, 64, 4095); + l.unback_to(l.keep_chunks()); + assert_eq!( + l.reclaim_delay(2, 32, 0, s), + None, + "armed with nothing grown" + ); + } + /// A client-boundary re-initialization with 23 chunks backed and no render after it: the - /// kept backing is due to be freed after the long idle, and once freed nothing more is due. + /// kept backing is due to be freed after the long idle, measured from the start of the idle, + /// and once freed nothing more is due. #[test] fn pending_reinit_frees_kept_backing() { let s = 1_000_000_000; @@ -731,10 +853,19 @@ mod tests { l.unback_to(23); l.posted = 2800; assert!(l.drain_reinit_due(DRAIN_IDLE_NS)); + // The poll after the 50 ms re-initialization waits for the rest of the long idle. + assert_eq!( + l.pending_free_delay(2, DRAIN_IDLE_NS, s), + Some(s - DRAIN_IDLE_NS) + ); assert!(!l.pending_free_due(s - 1, s)); assert!(l.pending_free_due(s, s)); + assert_eq!(l.pending_free_delay(2, s + 1, s), Some(0)); + assert_eq!(l.pending_free_delay(1, 0, s), None); + assert_eq!(l.pending_free_delay(0, 0, s), None); l.unback_to(l.keep_chunks()); assert!(!l.pending_free_due(100 * s, s)); + assert_eq!(l.pending_free_delay(2, 0, s), None); } #[test] diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 43949e9d91be8..3bd600790b0e3 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -3,17 +3,17 @@ """Run the G15 PB growth answer and its on-demand backing, actual bodies, against a model. The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), -answer_request(), map_pb(), refill_backing(), note_idle(), reclaim_check() and free_backing(), -Pools::soft_max(), map_heap(), free_chunks() and back_chunk(), PbChunk, -chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from vm.rs -VmState::map_one(), map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks(), -pb_blocks(), pb_reclaim(), pb_idle_ns() and pb_reclaim_blocks() from params.rs; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and -is_cached_noncoherent() from pgtable.rs; and the parameters' defaults from asahi_core.rs. It -includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model -of the firmware objects, the GEM backing allocation, the shared layer's map_at() and the DEVCTL -wait, which logs every allocation, alias map, unmap, free, block record, barrier, producer -store, reply and log line in order. Allocation and alias map failures can be injected. Source -checks: publish_prefix_into() starts with the PB catch-up, and pools.rs asserts the PB leaf +answer_request(), map_pb(), refill_backing(), note_idle(), reclaim_check(), free_backing() and +reclaim_delay_ns(), Pools::soft_max(), map_heap(), free_chunks() and back_chunk(), PbChunk, +chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from vm.rs VmState::map_one(), +map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks(), pb_blocks(), pb_reclaim(), +pb_idle_ns() and pb_reclaim_blocks() from params.rs; the PB sizes, tuning and PB leaf word from +hw.rs; Prot::from_pte() and is_cached_noncoherent() from pgtable.rs; and the parameters' defaults +from asahi_core.rs. It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies +run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() +and the DEVCTL wait, which logs every allocation, alias map, unmap, free, block record, barrier, +producer store, reply and log line in order. Allocation and alias map failures can be injected. +Source checks: publish_prefix_into() starts with the PB catch-up, and pools.rs asserts the PB leaf GPU-only at compile time. What it shows: @@ -46,8 +46,11 @@ client that exits with nothing rendered after it is re-initialized after 50 ms with the backing kept, and the kept backing is freed after the long idle, its posted count kept until the InitBM; at reclaim level 1 the backing stays; a recovery's pending re-initialization frees nothing; + driven as the idle poll work arms it (reclaim_delay_ns()), the same exit wakes at 50 ms and at + the long idle and arms nothing after; an idle reclaim arms nothing once its re-initialization + is pending; - nothing grown since the initialization (an initial posting above half the maximum, or a - reclaim floor below it): no re-initialization; + reclaim floor below it): no re-initialization and nothing armed; - the actual reclaim keeps the posted count until the next render's InitBM (a model of the publication: map_pb() first, then for a render prefix with a re-initialization pending the ledger reset of Pools::init()): the same client rendering again after 60 ms re-posts the @@ -211,12 +214,49 @@ ' || self.engine.frozen\n' ' || self.recovery.failed().is_some()\n' ' {\n return;\n'), + # Both checks stop on any pending re-initialization: the code before the kept-backing free. + 'old-reinit-guards': [(POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return;\n'), + (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n')], + # Arm nothing while a reclaim's re-initialization is pending. + 'pending-reinit-arms-nothing': (POOLS, ' || (self.recovery.reinit && !self.pools.reclaim_pending)\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n', + ' || self.recovery.reinit\n' + ' || self.recovery.armed.is_some()\n' + ' || self.engine.frozen\n' + ' || self.recovery.failed().is_some()\n' + ' {\n return None;\n'), + # Arm the poll for the expired 50 ms instead of the rest of the long idle. + 'pending-delay-from-drain': (GROWTH, 'Some(min_idle_ns.saturating_sub(idle_ns))', + 'Some(DRAIN_IDLE_NS.saturating_sub(idle_ns))'), # Re-initialize at an idle with nothing grown (posted above the floor only). 'reclaim-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > floor && idle_ns >= min_idle_ns', 'self.posted > floor && idle_ns >= min_idle_ns'), # Re-initialize at a client boundary with nothing grown (initial above half the maximum). 'drain-due-nothing-grown': (GROWTH, 'self.posted > self.initial && self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS', 'self.posted > self.max / 2 && idle_ns >= DRAIN_IDLE_NS'), + # Arm the poll with nothing grown. + 'delay-nothing-grown': (GROWTH, ' let grown = self.posted > self.initial;\n', ' let grown = true;\n'), # Reset the posted count at the idle reclaim instead of at the next render's InitBM. 'reinit-at-idle-reclaim': (POOLS, ' if level >= 2 && idle {\n freed = self.free_backing();\n', ' if level >= 2 && idle {\n self.pools.ledger.reinit();\n freed = self.free_backing();\n'), @@ -261,12 +301,16 @@ 'refill-skip-with-live-render': ('pending_reinit_with_a_live_render_refills', 'growth request stranded'), 'refill-for-any-flight': ('pending_reinit_with_compute_only_backs_nothing', 'backed for a compute-only flight'), 'map-without-identity': ('freed_chunks_are_mapped_again', 'alias of another object'), - 'pending-reinit-stops-free': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), + 'pending-reinit-stops-free': ('client_boundary_reinit_arms_the_long_idle', 'poll woke in a loop'), + 'old-reinit-guards': ('client_boundary_reinit_arms_the_long_idle', 'kept backing not freed'), + 'pending-reinit-arms-nothing': ('client_boundary_reinit_arms_the_long_idle', 'kept backing not freed'), + 'pending-delay-from-drain': ('client_boundary_reinit_arms_the_long_idle', 'poll woke in a loop'), 'pending-free-at-level-1': ('client_boundary_reinit_at_level_1_keeps_the_backing', 'freed at level 1'), 'reclaim-pending-not-set': ('client_boundary_reinit_frees_without_a_render', 'kept backing not freed'), 'free-on-recovery-reinit': ('recovery_reinit_frees_nothing', 'freed for a recovery'), 'reclaim-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (64 blocks, floor 32)'), 'drain-due-nothing-grown': ('nothing_grown_reinitializes_nothing', 're-initialization with nothing grown (3000 blocks'), + 'delay-nothing-grown': ('nothing_grown_arms_nothing', 'poll woke in a loop'), 'reinit-at-idle-reclaim': ('same_client_render_after_1_1s', 'posted count reset before the InitBM (1500)'), 'reinit-at-any-reclaim': ('same_client_render_after_60ms', 'posted count reset before the InitBM'), 'freed-without-walk': ('failed_vm_walk_counts_nothing_freed', 'freed counted with the VM walk failed'), @@ -1793,6 +1837,7 @@ def consts(text, names): check_at(&mut f, t0); check_at(&mut f, t0 + 50_000_000); assert!(f.recovery.reinit); + assert_eq!(f.reclaim_delay_ns(), None, "armed at level 1"); check_at(&mut f, t0 + 5_000_000_000); assert_eq!(f.pools.chunk_count(), chunks, "freed at level 1"); assert_eq!(f.pools.stats.freed, 0); @@ -1831,11 +1876,102 @@ def consts(text, names): f.recovery.reinit = true; let t0 = now(); check_at(&mut f, t0); + assert_eq!(f.reclaim_delay_ns(), None, "armed for a recovery's re-initialization"); check_at(&mut f, t0 + 5_000_000_000); assert_eq!(f.pools.chunk_count(), chunks, "freed for a recovery's re-initialization"); assert_eq!(f.pools.stats.reclaims, 0); } + /// The idle poll work as poll.rs drives it: a pass with nothing published notes the idle + /// and runs the reclaim check; an idle pass is re-armed only for reclaim_delay_ns(), + /// rounded up to whole ms, plus one 4 ms jiffy. Returns the wake times. + fn idle_polls(f: &mut Firmware, limit: usize) -> Vec { + let mut wakes = Vec::new(); + loop { + f.note_idle(); + f.reclaim_check(); + let Some(ns) = f.reclaim_delay_ns() else { + return wakes; + }; + assert!(wakes.len() < limit, "poll woke in a loop: {} wakes, next in {ns} ns", wakes.len()); + let t = now() + ns.div_ceil(1_000_000) * 1_000_000 + 4_000_000; + crate::NOW.with(|n| n.set(t)); + wakes.push(t); + } + } + + /// The same exit with the idle poll work: the poll is armed for the 50 ms + /// re-initialization, then for the rest of the long idle (not the expired 50 ms), where the + /// kept backing is freed; then nothing is armed. + #[test] + fn client_boundary_reinit_arms_the_long_idle() { + let mut f = Firmware::new(); + grow_to(&mut f, 2100); + let (posted, chunks) = (f.pools.ledger.posted, f.pools.chunk_count()); + assert!(posted > 2047 && chunks > 2); + // The pass that retires the last flight notes the idle. + f.engine.flights.clear(); + f.note_idle(); + let t0 = now(); + take(); + let wakes = idle_polls(&mut f, 8); + let log = take(); + assert_eq!( + f.pools.chunk_count(), + 2, + "kept backing not freed: {} chunks, wakes {:?} ms", + f.pools.chunk_count(), + wakes.iter().map(|t| (t - t0) / 1_000_000).collect::>() + ); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + assert_eq!(f.pools.stats.reclaims, 1); + assert_eq!(f.pools.ledger.backed, 2 * CHUNK_BLOCKS); + assert_eq!(f.pools.stats.freed, (chunks - 2) as u64); + assert_eq!(f.pools.ledger.posted, posted, "posted count reset before the InitBM"); + // Two wakes: the re-initialization after 50 ms, the free after the long idle. + let ms: Vec = wakes.iter().map(|t| (t - t0) / 1_000_000).collect(); + assert_eq!(ms, vec![54, 1004], "wakes (ms after the idle began)"); + assert_eq!(count(&log, |e| matches!(e, Ev::Unmap(5, _))), chunks - 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_))), chunks - 2); + // Idle much longer, or a job arriving: nothing more is due. + crate::NOW.with(|n| n.set(t0 + 100_000_000_000)); + f.reclaim_check(); + assert_eq!(f.reclaim_delay_ns(), None); + assert!(take().is_empty()); + } + + /// Nothing grown: the idle poll work arms nothing. + #[test] + fn nothing_grown_arms_nothing() { + for (blocks, floor) in [(64, 32), (3000, 1024)] { + defaults(); + crate::PARAM_BLOCKS.with(|b| b.set(blocks)); + crate::PARAM_RECLAIM_BLOCKS.with(|b| b.set(floor)); + let mut f = Firmware::new(); + f.engine.flights.clear(); + let wakes = idle_polls(&mut f, 8); + assert!(wakes.is_empty(), "armed with nothing grown ({blocks} blocks, floor {floor}): {} wakes", wakes.len()); + assert!(!f.recovery.reinit); + } + } + + /// An idle reclaim below half the maximum frees at the long idle; with its re-initialization + /// pending nothing more is armed. + #[test] + fn idle_reclaim_arms_nothing_after() { + let mut f = Firmware::new(); + grow_to(&mut f, 1500); + let chunks = f.pools.chunk_count(); + f.engine.flights.clear(); + f.note_idle(); + let t0 = now(); + let wakes = idle_polls(&mut f, 8); + let ms: Vec = wakes.iter().map(|t| (t - t0) / 1_000_000).collect(); + assert_eq!(ms, vec![1004]); + assert_eq!((f.pools.chunk_count(), f.pools.stats.freed), (2, (chunks - 2) as u64)); + assert!(f.recovery.reinit && f.pools.reclaim_pending); + } + #[test] fn catch_up_refuses_flights_of_another_vm() { let mut f = Firmware::new(); @@ -1939,7 +2075,8 @@ def run(mutation): ' pub(crate) fn refill_backing(&mut self) {', ' pub(crate) fn note_idle(&mut self) {', ' pub(crate) fn reclaim_check(&mut self) {', - ' fn free_backing(&mut self) -> usize {')] + ' fn free_backing(&mut self) -> usize {', + ' pub(crate) fn reclaim_delay_ns(&self) -> Option {')] soft_max = item(pools, ' pub(crate) fn soft_max(&self) -> u32 {') pools_methods = (item(pools, ' fn map_heap(&self, vm: &VmState) -> Result {') + '\n' + item(pools, ' fn free_chunks(&mut self, keep: usize) -> usize {') + '\n' + From 2c428477bc20b62c075b26bcf1635dbc55db8b54 Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Fri, 23 Oct 2026 02:15:20 +0530 Subject: [PATCH 09/46] drm/asahi: g15: say when the boot loader left the GPU node disabled The boot loader enables the T6030/T6031 GPU node only when its chosen.asahi,t603x-gpu setting is on. With the setting off, as the installed m1n1 configuration of a J516C can have it (chosen.asahi,t6031-gpu=0), the node stays disabled. The adapter then never probes, and the kernel log gives no reason why there is no native G15 GPU. At module activation, log one line when /chosen carries asahi,t6031-gpu or asahi,t6030-gpu and the matching GPU node is disabled. The line names the node and the property's value. Nothing else changes; machines without the property log nothing. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit a089fd06488aa5edb09674a8d3ca71b56db07087) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/boot.rs | 1 + drivers/gpu/drm/asahi/g15/glue.c | 42 +++++++++++++++++++++++++++++ drivers/gpu/drm/asahi/g15/mod.rs | 2 ++ tools/asahi/g15-probe-init-tests.py | 2 ++ 4 files changed, 47 insertions(+) diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index 7a06aa64ffbc3..d0dbe139efbe6 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -63,6 +63,7 @@ pub(crate) mod glue { fn asahi_g15_release_power(dev: *mut core::ffi::c_void); fn asahi_g15_module_get(); pub(crate) fn asahi_g15_reprobe(); + pub(crate) fn asahi_g15_report_loader_gpu(); fn asahi_g15_adt_phram(start: *mut u64, size: *mut u64) -> core::ffi::c_int; fn asahi_g15_ram_disjoint(start: u64, size: u64) -> bool; fn asahi_g15_ram_within(start: u64, size: u64) -> bool; diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index b949a4b9b61ed..f61f9559ee0a2 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -28,6 +28,7 @@ void asahi_g15_hold_power(struct device *dev); void asahi_g15_release_power(struct device *dev); void asahi_g15_module_get(void); void asahi_g15_reprobe(void); +void asahi_g15_report_loader_gpu(void); int asahi_g15_adt_phram(u64 *start, u64 *size); bool asahi_g15_ram_disjoint(u64 start, u64 size); bool asahi_g15_ram_within(u64 start, u64 size); @@ -39,6 +40,47 @@ int asahi_g15_pmp_get(struct device *dev); void asahi_g15_pmp_put(struct device *dev); bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); +/* + * The boot loader enables the T6030/T6031 GPU node only when its + * chosen.asahi,t603x-gpu setting is on; otherwise the node stays disabled, + * the adapter never probes and nothing says why native G15 is absent. Name + * the setting once, at module activation, on a machine whose /chosen has it + * and whose GPU node is disabled. Nothing else changes. + */ +void asahi_g15_report_loader_gpu(void) +{ + static const struct { + const char *prop; + const char *compatible; + } socs[] = { + { "asahi,t6031-gpu", "apple,agx-t6031" }, + { "asahi,t6030-gpu", "apple,agx-t6030" }, + }; + struct device_node *chosen, *np; + const char *value; + unsigned int i; + u32 cell; + + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return; + for (i = 0; i < ARRAY_SIZE(socs); i++) { + if (!of_property_present(chosen, socs[i].prop)) + continue; + np = of_find_compatible_node(NULL, NULL, socs[i].compatible); + if (np && !of_device_is_available(np)) { + if (!of_property_read_string(chosen, socs[i].prop, &value)) + pr_info("asahi: G15: native G15 not started: the boot loader left the GPU node %pOF disabled (/chosen/%s = \"%s\")\n", + np, socs[i].prop, value); + else if (!of_property_read_u32(chosen, socs[i].prop, &cell)) + pr_info("asahi: G15: native G15 not started: the boot loader left the GPU node %pOF disabled (/chosen/%s = %u)\n", + np, socs[i].prop, cell); + } + of_node_put(np); + } + of_node_put(chosen); +} + /* * The maintained m1n1 handoff keeps these four UAT regions in the linear map. * Require the exact enabled reserved-memory owner, in addition to the ADT diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 713a1360b5687..f709c9ed76f42 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -226,6 +226,8 @@ static MODULE_READY: AtomicBool = AtomicBool::new(false); /// No fallible module initialization may follow this call. pub(crate) fn activate() { MODULE_READY.store(true, Ordering::Release); + // SAFETY: The helper only reads the device tree. + unsafe { boot::glue::asahi_g15_report_loader_gpu() }; // SAFETY: Both pinned platform registrations are fully initialized and remain owned // by DriverModule. The C helper borrows their live driver pointer only for this call. unsafe { boot::glue::asahi_g15_reprobe() }; diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py index 65cc9c0b69c9f..e7617fba021c2 100644 --- a/tools/asahi/g15-probe-init-tests.py +++ b/tools/asahi/g15-probe-init-tests.py @@ -193,6 +193,8 @@ def item(text, marker): mod boot { pub mod glue { pub fn retained() -> bool { crate::RETAINED.with(core::cell::Cell::get) } pub unsafe fn asahi_g15_reprobe() { crate::activation_reprobe(); } + // Reads the device tree and logs; no driver state. + pub unsafe fn asahi_g15_report_loader_gpu() {} }} ''' From bb1b0c80c14f87c0203291a12bd6fc00a4eadd41 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:17:28 +1100 Subject: [PATCH 10/46] tools/asahi: run-host-tests: give device-tree tests a built dtc The T6030 PMP device-tree test that follows compiles the board trees with this kernel's scripts/dtc/dtc and reads the result with fdtget. Build dtc next to the kconfig tool the runner already builds, pass it to a test as '{dtc}', and let --dtc name an existing one. The CI job installs device-tree-compiler when fdtget is missing. Assisted-by: LLM Signed-off-by: Ryan Murray --- .github/workflows/host-tests.yml | 2 ++ tools/asahi/run-host-tests.py | 25 ++++++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/host-tests.yml b/.github/workflows/host-tests.yml index f16fb33953f7c..fd665f948d991 100644 --- a/.github/workflows/host-tests.yml +++ b/.github/workflows/host-tests.yml @@ -57,6 +57,8 @@ jobs: command -v "$p" >/dev/null || need="$need $p" done command -v "ld.lld-$CLANG_VERSION" >/dev/null || need="$need lld-$CLANG_VERSION" + # Device-tree tests read the built trees with fdtget. + command -v fdtget >/dev/null || need="$need device-tree-compiler" dpkg -s "libclang-rt-$CLANG_VERSION-dev" >/dev/null 2>&1 || need="$need libclang-rt-$CLANG_VERSION-dev" if [ -n "$need" ]; then sudo apt-get update -q diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 318f9e892505c..2ade9c97f854b 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -14,6 +14,9 @@ Negative controls read older revisions with `git show`, so the checkout needs the commits (a full or treeless clone, not a depth-1 one). +A test that reads device trees gets '{dtc}', this tree's scripts/dtc/dtc, built +with the kconfig tool unless --dtc names one. + Usage: tools/asahi/run-host-tests.py [--jobs N] [--work DIR] [--only SUBSTR] """ import argparse @@ -168,7 +171,7 @@ def mutations(flag, names, base, expect=KILL): ] -def jobs(work, conf): +def jobs(work, conf, dtc): found = {p.name for p in ASAHI.iterdir() if re.search(r'-tests\.(py|rs|sh)$', p.name) and p.name != Path(__file__).name} known = set(ASAHI_TESTS) | set(ASAHI_RUST) | set(ASAHI_SHELL) @@ -190,9 +193,11 @@ def jobs(work, conf): for name, runs in ASAHI_TESTS.items(): for index, (args, expect) in enumerate(runs): label = name.removesuffix('.py') + ('' if not args or args == out() or args == out('--output') - else ' ' + ' '.join(a for a in args if a not in ('{out}', '--out', '--output', '{conf}', '--conf'))) + else ' ' + ' '.join(a for a in args if a not in ('{out}', '--out', '--output', '{conf}', '--conf', + '{dtc}', '--dtc'))) d = work / f'{name}.{index}' - cmd = ['python3', '-I', '-B', str(ASAHI / name)] + [a.replace('{out}', str(d / 'out')).replace('{conf}', str(conf)) for a in args] + cmd = ['python3', '-I', '-B', str(ASAHI / name)] + [a.replace('{out}', str(d / 'out')).replace('{conf}', str(conf)) + .replace('{dtc}', str(dtc)) for a in args] result.append((label, cmd, ASAHI, expect, d, {})) for name in ASAHI_RUST: d = work / name @@ -233,6 +238,7 @@ def main(): ap.add_argument('--jobs', type=int, default=os.cpu_count() or 2) ap.add_argument('--work', type=Path, help='keep logs and build products here (default: a temporary directory)') ap.add_argument('--conf', type=Path, help='a built scripts/kconfig/conf (default: build one)') + ap.add_argument('--dtc', type=Path, help='a built scripts/dtc/dtc (default: build one)') ap.add_argument('--only', help='run only jobs whose name contains this text') ap.add_argument('--timeout', type=int, default=900) args = ap.parse_args() @@ -240,13 +246,18 @@ def main(): work = args.work or Path(tempfile.mkdtemp(prefix='aurora-host-tests-')) work = work.resolve() work.mkdir(parents=True, exist_ok=True) - conf = args.conf - if conf is None: - build = work / 'kconfig' + conf, dtc = args.conf, args.dtc + build = work / 'kconfig' + if conf is None or dtc is None: subprocess.run(['make', '-s', '-C', str(ROOT), f'O={build}', 'ARCH=arm64', 'allnoconfig'], check=True, stdout=subprocess.DEVNULL) + if conf is None: conf = build / 'scripts/kconfig/conf' - todo = jobs(work, conf.resolve()) + if dtc is None: + subprocess.run(['make', '-s', '-C', str(ROOT), f'O={build}', 'ARCH=arm64', 'scripts_dtc'], + check=True, stdout=subprocess.DEVNULL) + dtc = build / 'scripts/dtc/dtc' + todo = jobs(work, conf.resolve(), dtc.resolve()) if args.only: todo = [j for j in todo if args.only in j[0]] print(f'{len(todo)} runs, {args.jobs} at a time, logs in {work}', flush=True) From bc4f74455fcd84cfe6d2ec3a7a1bad484dfe211b Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Sat, 24 Oct 2026 16:34:27 +0200 Subject: [PATCH 11/46] arm64: dts: apple: t6030: guard static PMP nodes The static PMP nodes block the display gate's overlay and leave G15 deferred on M3 Pro. Keeping either the static DART or mailbox would still conflict with the overlay. Guard all four nodes with APPLE_USE_PMP, matching the existing alias and consumer guards. Check the default and static PMP nodes in CI. Assisted-by: Codex:GPT-6 Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Eryk Wieliczko (cherry picked from commit 2c916d27ae4eba6d47cb92277a00e9a4ff946f4b) [ryan: register the test in tools/asahi/run-host-tests.py, with the runner's built dtc, instead of omacom's tools/aurora-ci/host-tests.json] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6030.dtsi | 4 ++ tools/asahi/run-host-tests.py | 1 + tools/asahi/t6030-pmp-dt-tests.py | 65 ++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 tools/asahi/t6030-pmp-dt-tests.py diff --git a/arch/arm64/boot/dts/apple/t6030.dtsi b/arch/arm64/boot/dts/apple/t6030.dtsi index 685b67606dc87..c19d3691f9227 100644 --- a/arch/arm64/boot/dts/apple/t6030.dtsi +++ b/arch/arm64/boot/dts/apple/t6030.dtsi @@ -836,6 +836,8 @@ #clock-cells = <1>; }; + /* The display handoff gate installs PMP unless explicitly static. */ + #ifdef APPLE_USE_PMP pmp_dart: iommu@350300000 { compatible = "apple,t6030-dart", "apple,t8110-dart"; reg = <0x3 0x50300000 0x0 0x4000>; @@ -985,6 +987,8 @@ power-domains = <&ps_pmp>, <&ps_pms_sram>; }; + #endif /* APPLE_USE_PMP */ + aic: interrupt-controller@351000000 { compatible = "apple,t6030-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 2ade9c97f854b..21e1f7f1f8fbe 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -139,6 +139,7 @@ def mutations(flag, names, base, expect=KILL): 't6030-clock-tests.py': [(out(), PASS)], 't6030-display-admission-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-display-linkage-tests.py': [(out('--output') + ['--conf', '{conf}'], PASS)], + 't6030-pmp-dt-tests.py': [(out('--output') + ['--dtc', '{dtc}'], PASS)], 't6030-pmp-graph-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-pmp-supplier-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], 't6030-wiring-tests.py': [(out(), PASS), (out() + ['--old-classifier', 'tb'], PASS), diff --git a/tools/asahi/t6030-pmp-dt-tests.py b/tools/asahi/t6030-pmp-dt-tests.py new file mode 100644 index 0000000000000..22826e998173d --- /dev/null +++ b/tools/asahi/t6030-pmp-dt-tests.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Check static PMP ownership in the compiled T6030 device trees.""" + +import argparse +from pathlib import Path +import subprocess + + +ROOT = Path(__file__).resolve().parents[2] +BOARDS = ("t6030-j514s", "t6030-j516s") +PMP_NODES = { + "iommu@350300000", + "pmp_report@3503c0000", + "pmp@350500000", + "mbox@350c08000", +} + + +def check_profile(tree, output, dtc, board, static): + profile = "static" if static else "default" + dtb = output / f"{board}-{profile}.dtb" + command = [ + "cpp", "-nostdinc", "-undef", "-D__DTS__", + "-I", str(tree / "scripts/dtc/include-prefixes"), + "-x", "assembler-with-cpp", + ] + if static: + command.append("-DAPPLE_USE_PMP") + command.append(str(tree / f"arch/arm64/boot/dts/apple/{board}.dts")) + source = subprocess.check_output(command) + subprocess.run([dtc, "-q", "-I", "dts", "-O", "dtb", "-o", str(dtb)], + input=source, check=True) + + children = subprocess.check_output(["fdtget", "-l", str(dtb), "/soc"], text=True) + present = PMP_NODES.intersection(children.split()) + expected = PMP_NODES if static else set() + if present != expected: + raise AssertionError(f"{board} {profile}: static PMP nodes " + f"{sorted(present)}, expected {sorted(expected)}") + + alias = subprocess.run(["fdtget", "-t", "s", str(dtb), "/aliases", "pmp"], + capture_output=True, text=True) + if static: + if alias.returncode or alias.stdout.strip() != "/soc/pmp@350500000": + raise AssertionError(f"{board} {profile}: missing or incorrect PMP alias") + elif alias.returncode == 0: + raise AssertionError(f"{board} {profile}: unexpected static PMP alias") + print(f"{board} {profile}: PASS") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tree", type=Path, default=ROOT) + parser.add_argument("--dtc", required=True, help="this kernel's built scripts/dtc/dtc") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + for board in BOARDS: + for static in (False, True): + check_profile(args.tree.resolve(), args.output, args.dtc, board, static) + + +if __name__ == "__main__": + main() From aaba39eef4aea4d6b6625ced8fb2269f09b615e4 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Sat, 24 Oct 2026 16:34:28 +0200 Subject: [PATCH 12/46] drm/asahi: g15: match UAT reservations with unit addresses The DT uses names such as uat-ttbs@10, but of_find_node_by_path() requires an exact match and misses them. Use of_node_name_eq() on /reserved-memory children. Require one matching owner and keep the existing range and property checks. Extend the host test to cover unit addresses and duplicate owners. Assisted-by: Codex:GPT-6 Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Eryk Wieliczko (cherry picked from commit 8b0cbd58bf95692ae58f7fd88e7e2b57e71cadc3) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/glue.c | 29 ++++-- .../drivers/gpu/asahi-g15/uat-owner.c | 95 ++++++++++++++++--- 2 files changed, 104 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index f61f9559ee0a2..8507ae3796a00 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -88,20 +88,31 @@ void asahi_g15_report_loader_gpu(void) */ bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size) { - static const char *const paths[] = { - "/reserved-memory/uat-ttbs", - "/reserved-memory/uat-pagetables", - "/reserved-memory/uat-handoff", - "/reserved-memory/uat-pagetables-l2", + static const char *const names[] = { + "uat-ttbs", + "uat-pagetables", + "uat-handoff", + "uat-pagetables-l2", }; - struct device_node *np; + struct device_node *rmem, *child, *np = NULL; struct resource res; + unsigned int count = 0; bool match; - if (index >= ARRAY_SIZE(paths)) + if (index >= ARRAY_SIZE(names)) return false; - np = of_find_node_by_path(paths[index]); - match = np && of_device_is_available(np) && + + /* Unit addresses belong to the DT layout, not the reservation name. */ + rmem = of_find_node_by_path("/reserved-memory"); + for_each_child_of_node(rmem, child) { + if (!of_node_name_eq(child, names[index])) + continue; + count++; + if (!np) + np = of_node_get(child); + } + of_node_put(rmem); + match = count == 1 && np && of_device_is_available(np) && !of_property_read_bool(np, "reusable") && !of_get_address(np, 1, NULL, NULL) && !of_address_to_resource(np, 0, &res) && diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c b/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c index 9909ac31e3a41..48e705c60b77f 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c @@ -8,22 +8,58 @@ typedef uint32_t u32; typedef uint64_t u64; struct device_node { + const char *name; + struct device_node *child, *next; bool available, reusable, extra, bad_reg; + u64 start, size; + int refs; }; struct resource { u64 start, end; }; -static struct device_node node = { .available = true }; +static struct device_node nodes[5]; +static struct device_node rmem; +#define node nodes[0] static bool missing; static u64 start = 0x100000, size = 0x4000; -static int refs; +static inline struct device_node *of_node_get(struct device_node *np) +{ + if (np) + np->refs++; + return np; +} +static void of_node_put(struct device_node *np); static struct device_node *of_find_node_by_path(const char *path) { - assert(!strncmp(path, "/reserved-memory/uat-", 21)); + const char *prefix = "/reserved-memory/"; + if (missing) return NULL; - refs++; - return &node; + if (!strcmp(path, "/reserved-memory")) + return of_node_get(&rmem); + assert(!strncmp(path, prefix, strlen(prefix))); + for (struct device_node *np = rmem.child; np; np = np->next) + if (!strcmp(np->name, path + strlen(prefix))) + return of_node_get(np); + return NULL; +} +static inline struct device_node *of_get_next_child(struct device_node *parent, + struct device_node *prev) +{ + struct device_node *next = NULL; + + if (parent) + next = prev ? prev->next : parent->child; + of_node_put(prev); + return of_node_get(next); +} +#define for_each_child_of_node(parent, child) \ + for ((child) = of_get_next_child((parent), NULL); (child); \ + (child) = of_get_next_child((parent), (child))) +static inline bool of_node_name_eq(struct device_node *np, const char *name) +{ + return np && strlen(name) == strcspn(np->name, "@") && + !strncmp(np->name, name, strlen(name)); } static bool of_device_is_available(struct device_node *np) { @@ -44,8 +80,8 @@ static int of_address_to_resource(struct device_node *np, int index, struct resource *res) { assert(index == 0); - res->start = start; - res->end = start + size - 1; + res->start = np->start; + res->end = np->start + np->size - 1; return np->bad_reg ? -1 : 0; } static u64 resource_size(struct resource *res) @@ -55,14 +91,49 @@ static u64 resource_size(struct resource *res) static void of_node_put(struct device_node *np) { if (np) - refs--; + assert(np->refs-- > 0); } #include "uat-production.h" int main(void) { + static const char *const names[] = { + "uat-ttbs", "uat-pagetables", "uat-handoff", "uat-pagetables-l2" + }; + static const char *const unit_names[] = { + "uat-ttbs@10", "uat-pagetables@20", "uat-handoff@30", "uat-pagetables-l2@40" + }; + rmem.child = nodes; + for (u32 i = 0; i < 4; ++i) { + nodes[i].available = true; + nodes[i].start = start + i * size; + nodes[i].size = size; + nodes[i].name = names[i]; + nodes[i].next = i < 3 ? &nodes[i + 1] : NULL; + } + for (u32 i = 0; i < 4; ++i) + assert(asahi_g15_reserved_uat(i, nodes[i].start, size)); for (u32 i = 0; i < 4; ++i) - assert(asahi_g15_reserved_uat(i, start, size)); + nodes[i].name = unit_names[i]; + for (u32 i = 0; i < 4; ++i) + assert(asahi_g15_reserved_uat(i, nodes[i].start, size)); + nodes[3].next = &nodes[4]; + nodes[4].name = "uat-ttbs@99"; + nodes[4].available = true; + assert(!asahi_g15_reserved_uat(0, start, size)); + nodes[4].available = false; + assert(!asahi_g15_reserved_uat(0, start, size)); + nodes[3].next = NULL; + node.name = "uat-ttbs-other@10"; + assert(!asahi_g15_reserved_uat(0, start, size)); + node.name = unit_names[0]; + for (u32 i = 0; i < 4; ++i) { + const char *saved = nodes[i].name; + + nodes[i].name = "unrelated@10"; + assert(!asahi_g15_reserved_uat(i, nodes[i].start, size)); + nodes[i].name = saved; + } assert(!asahi_g15_reserved_uat(4, start, size)); assert(!asahi_g15_reserved_uat(0, start + 0x4000, size)); assert(!asahi_g15_reserved_uat(0, start, size + 0x4000)); @@ -80,6 +151,8 @@ int main(void) node.extra = false; node.bad_reg = true; assert(!asahi_g15_reserved_uat(0, start, size)); - assert(refs == 0); - puts("production UAT owner: exact ranges, unavailable/reusable/malformed/extra reservations: PASS"); + assert(rmem.refs == 0); + for (u32 i = 0; i < ARRAY_SIZE(nodes); ++i) + assert(nodes[i].refs == 0); + puts("production UAT owner: unit addresses, unique owners and exact ranges: PASS"); } From 1cf552707286b98140a33a23945f16c594df2717 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Mon, 26 Oct 2026 03:17:58 +0000 Subject: [PATCH 13/46] drm/asahi: g15: pin the module with try_module_get before firmware start A started G15 firmware session cannot be stopped, so boot::start pins the module before its first firmware-visible write and never drops the reference. The pin used __module_get(), which increments the count unconditionally, even after an ordinary rmmod has passed its reference check in try_stop_module(). The first start can run while the module is LIVE. Activation attaches synchronously during module init, but a probe deferred for the PMP supplier, or attached asynchronously with async_probe, runs later from the deferred-probe or async worker. If an rmmod has passed its reference check by then, module exit unregisters the driver, which waits for the in-flight probe to finish starting the firmware, unbinds the device and frees the module under the live session and its RTKit callbacks. Take the reference with try_module_get() before RETAINED is set and before anything firmware-visible. Once removal has begun this fails with ENODEV and leaves the firmware untouched, cleaned up as for a failed self-check: the cached-class objects are forgotten, the PMP vote is released and the session stays unretained. A caller that loses the race to set RETAINED drops only its own reference. The policy is in g15/retain.rs so the asahi-g15 host runner covers it: a refused pin marks nothing, the first pin is kept and later calls take nothing, a lost race drops only its own reference, and barrier-synchronized concurrent pins leave exactly one reference. Generated-by: LLM Co-Authored-By: Claude Opus 5.5 Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> (cherry picked from commit 095edba0d1bd43441369e09ab62e20be2c907ead) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/boot.rs | 30 ++-- drivers/gpu/drm/asahi/g15/glue.c | 19 ++- drivers/gpu/drm/asahi/g15/mod.rs | 10 +- drivers/gpu/drm/asahi/g15/retain.rs | 136 ++++++++++++++++++ .../selftests/drivers/gpu/asahi-g15/run.py | 1 + 5 files changed, 180 insertions(+), 16 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g15/retain.rs diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index d0dbe139efbe6..95f5238fa805a 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -61,7 +61,8 @@ pub(crate) mod glue { fn asahi_g15_machine(compatible: *const core::ffi::c_char) -> bool; fn asahi_g15_hold_power(dev: *mut core::ffi::c_void); fn asahi_g15_release_power(dev: *mut core::ffi::c_void); - fn asahi_g15_module_get(); + fn asahi_g15_module_try_get() -> bool; + fn asahi_g15_module_put(); pub(crate) fn asahi_g15_reprobe(); pub(crate) fn asahi_g15_report_loader_gpu(); fn asahi_g15_adt_phram(start: *mut u64, size: *mut u64) -> core::ffi::c_int; @@ -121,12 +122,17 @@ pub(crate) mod glue { RETAINED.load(Ordering::Acquire) } - /// Pins this module until reboot, once (`__module_get(THIS_MODULE)`). - pub(crate) fn retain_module() { - if !RETAINED.swap(true, Ordering::AcqRel) { - // SAFETY: The module is live while its code runs; the reference is never put. - unsafe { asahi_g15_module_get() }; - } + /// Pins this module until reboot, once (`try_module_get(THIS_MODULE)`, see `retain.rs`). + /// ENODEV, with nothing pinned or marked, once the module's removal has begun. + pub(crate) fn retain_module() -> Result { + crate::g15::retain::retain( + &RETAINED, + // SAFETY: Takes a reference to this module unless its removal has begun. + || unsafe { asahi_g15_module_try_get() }, + // SAFETY: Drops only the reference this call took; the winner's is never put. + || unsafe { asahi_g15_module_put() }, + ) + .map_err(|_| ENODEV) } /// The phram "adt" export: start and size. @@ -675,8 +681,16 @@ pub(crate) fn start( return Err(e); } + // Pin the module before anything firmware-visible. Once its removal has begun this fails + // and the firmware is left untouched, cleaned up as for a failed self-check: the vote is + // released and the session stays unretained. + if let Err(e) = glue::retain_module() { + dev_err!(d, "G15: the module is being removed; the firmware is not started\n"); + // Mapped with cached firmware classes: never unmapped. + core::mem::forget(objects); + return Err(e); + } // From here on the firmware is touched: nothing it can reach is freed, the vote included. - glue::retain_module(); vote.keep(); let rtkit = match Rtkit::new(&d, shared) { Ok(rtkit) => rtkit, diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index 8507ae3796a00..38359b39af3a7 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -26,7 +26,8 @@ bool asahi_g15_board(void); bool asahi_g15_machine(const char *compatible); void asahi_g15_hold_power(struct device *dev); void asahi_g15_release_power(struct device *dev); -void asahi_g15_module_get(void); +bool asahi_g15_module_try_get(void); +void asahi_g15_module_put(void); void asahi_g15_reprobe(void); void asahi_g15_report_loader_gpu(void); int asahi_g15_adt_phram(u64 *start, u64 *size); @@ -212,10 +213,20 @@ void asahi_g15_release_power(struct device *dev) } -/* Pin this module until reboot. */ -void asahi_g15_module_get(void) +/* + * Pin this module until reboot, unless its removal has begun: a deferred or + * asynchronous probe can run after an ordinary rmmod has passed its reference + * check, which __module_get() would not stop. + */ +bool asahi_g15_module_try_get(void) +{ + return try_module_get(THIS_MODULE); +} + +/* Drop a reference asahi_g15_module_try_get() took. */ +void asahi_g15_module_put(void) { - __module_get(THIS_MODULE); + module_put(THIS_MODULE); } /* diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index f709c9ed76f42..2a46b1637d03f 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -45,6 +45,7 @@ mod adt; mod board; mod boot; +mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -456,17 +457,18 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain(); + gpu.retain()?; Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once; `boot::start` already took it before touching the + /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) { - boot::glue::retain_module(); + pub(crate) fn retain(&self) -> Result { + boot::glue::retain_module() } } diff --git a/drivers/gpu/drm/asahi/g15/retain.rs b/drivers/gpu/drm/asahi/g15/retain.rs new file mode 100644 index 0000000000000..30fd79d510869 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/retain.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The once-per-boot module pin taken before the firmware is first touched. +//! +//! A started G15 firmware session cannot be stopped, so this module must stay loaded until +//! reboot from the first firmware-visible write on. `__module_get` alone does not ensure that: +//! a deferred or asynchronous probe can run while an ordinary rmmod is already past its +//! reference check, and the module is then unloaded under a running firmware. +//! `try_module_get` fails once removal has begun, so the pin is taken with it, before the +//! session is marked retained and before anything firmware-visible happens. + +use core::sync::atomic::{AtomicBool, Ordering}; + +/// The module is being removed: nothing was pinned or marked, and nothing may be started. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Removing; + +/// Pins the module once for the session `retained` tracks. `get` is +/// `try_module_get(THIS_MODULE)`, `put` is `module_put(THIS_MODULE)`. +/// +/// Already retained: nothing more is taken. Otherwise the reference comes first; `retained` is +/// set only once it is held. A caller that loses the race to set it drops only its own +/// reference, so exactly one reference stays and it is never put. +pub(crate) fn retain( + retained: &AtomicBool, + get: impl FnOnce() -> bool, + put: impl FnOnce(), +) -> Result<(), Removing> { + if retained.load(Ordering::Acquire) { + return Ok(()); + } + if !get() { + return Err(Removing); + } + if retained.swap(true, Ordering::AcqRel) { + put(); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use core::sync::atomic::AtomicI32; + use std::sync::Arc; + + #[test] + fn removal_fails_closed_with_nothing_pinned_or_marked() { + let retained = AtomicBool::new(false); + let refs = AtomicI32::new(1); // base reference only: rmmod has released it + let put = || panic!("nothing to put"); + assert_eq!(retain(&retained, || false, put), Err(Removing)); + assert!(!retained.load(Ordering::Acquire), "a refused pin must not mark the session"); + assert_eq!(refs.load(Ordering::Acquire), 1); + // A later live start can still pin: the refusal left no state behind. + let get = || { + refs.fetch_add(1, Ordering::AcqRel); + true + }; + assert_eq!(retain(&retained, get, || panic!("winner keeps its reference")), Ok(())); + assert!(retained.load(Ordering::Acquire)); + assert_eq!(refs.load(Ordering::Acquire), 2); + } + + #[test] + fn first_pin_is_kept_and_later_calls_take_nothing() { + let retained = AtomicBool::new(false); + let refs = AtomicI32::new(1); + let get = || { + refs.fetch_add(1, Ordering::AcqRel); + true + }; + assert_eq!(retain(&retained, get, || panic!("winner keeps its reference")), Ok(())); + // The post-start call in Gpu::retain and any later one: no new reference, no put. + for _ in 0..3 { + assert_eq!( + retain(&retained, || panic!("already retained"), || panic!("nothing to put")), + Ok(()) + ); + } + assert_eq!(refs.load(Ordering::Acquire), 2); + } + + #[test] + fn a_lost_race_drops_only_its_own_reference() { + let retained = AtomicBool::new(false); + let refs = AtomicI32::new(2); // base + the reference the winner already holds + let puts = AtomicI32::new(0); + // The winner marks the session between this caller's check and its swap. + let get = || { + refs.fetch_add(1, Ordering::AcqRel); + retained.store(true, Ordering::Release); + true + }; + let put = || { + refs.fetch_sub(1, Ordering::AcqRel); + puts.fetch_add(1, Ordering::AcqRel); + }; + assert_eq!(retain(&retained, get, put), Ok(())); + assert_eq!(puts.load(Ordering::Acquire), 1); + assert_eq!(refs.load(Ordering::Acquire), 2); + } + + #[test] + fn concurrent_pins_leave_exactly_one_reference() { + for _ in 0..500 { + let retained = Arc::new(AtomicBool::new(false)); + let refs = Arc::new(AtomicI32::new(1)); + let start = Arc::new(std::sync::Barrier::new(8)); + let threads: Vec<_> = (0..8) + .map(|_| { + let (retained, refs, start) = (retained.clone(), refs.clone(), start.clone()); + std::thread::spawn(move || { + let (a, b) = (refs.clone(), refs.clone()); + start.wait(); + retain( + &retained, + move || { + a.fetch_add(1, Ordering::AcqRel); + true + }, + move || { + b.fetch_sub(1, Ordering::AcqRel); + }, + ) + }) + }) + .collect(); + for t in threads { + assert_eq!(t.join().unwrap(), Ok(())); + } + assert!(retained.load(Ordering::Acquire)); + assert_eq!(refs.load(Ordering::Acquire), 2, "exactly one kept reference"); + } + } +} diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 861290ef24098..81883c315d27b 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -304,6 +304,7 @@ def main(): harness += f'#[path = "{G15 / "recovery/logic.rs"}"] mod recovery;\n' harness += f'#[path = "{G15 / "power/idle.rs"}"] mod idle;\n' harness += f'#[path = "{G15 / "power/pstate.rs"}"] mod pstate;\n' + harness += f'#[path = "{G15 / "retain.rs"}"] mod retain;\n' harness += ("mod firmware_identity {\n" + f'#[path = "{G15 / "identity/matching.rs"}"] pub mod matching;\n' + f'#[path = "{G15 / "identity/lookup.rs"}"] pub mod lookup;\n' + "}\n") From 068c2fa3a45a4533c9b27ec7deaac6885fc357fc Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Wed, 28 Oct 2026 04:06:37 +0530 Subject: [PATCH 14/46] drm/asahi: g15: t6031: build the J516C board data without a board file The J516C hardware record took 32 words from asahi/t6031-j516c-board.bin, a per-machine file that installs do not ship, and the driver compared the three images it built against the file before starting the firmware. A J516C without the file could not start the GPU. Take those 32 words from sources every J516C has, as the T6030 path does with its templates: - The six GPU sensor rows (18 words) are entries of the ADT's /arm-io/aon-ptd ptd_data_table, which the boot loader already exports (the phram "adt" region the T6030 path reads): the "RT" entries of kind g (four), r and f. The device tree does not carry the table. Any other count is refused. - The maximum power, the relative powers of DVFS states 1..7 and the MTR fast-die and alarm masks (14 words) are not in the device tree or the ADT. They become the J516C reference machine's values (board::record::J516C), as the T6030 templates carry their reference machine's. The device tree's operating-point powers stay unused; they are a boot-loader stand-in. hw::Board::Blob becomes hw::Board::DeviceTree(&Reference): no firmware request, no file digest, no file cross-check. The firmware identity pins and the apple,j516c machine match are unchanged. board/blob.rs and the RUST_FW_LOADER_ABSTRACTIONS select go. The off-tree J516C fixture test now builds the record from a saved device tree and ADT with the reference words and checks it against the machine's own record: byte-identical except the 42 words shown unneeded earlier. g15-zero-words-tests.py follows the new sources (14 reference + 18 sensor row + 42 zero words = the original 74); its hw.rs and prepare_board() pins are refreshed for the Board change, the T6030 arms are unchanged. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit aeffd0e53b4218633bd1f35fcdfe9e27188a5922) [ryan: our G15 Kconfig has no DRM_ASAHI_UAT_ADOPTION select; the zero-words harness keeps our docstring history and pins our hw.rs bytes] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/Kconfig | 1 - drivers/gpu/drm/asahi/g15/adt.rs | 57 +++- drivers/gpu/drm/asahi/g15/board.rs | 35 +- drivers/gpu/drm/asahi/g15/board/blob.rs | 113 ------ drivers/gpu/drm/asahi/g15/board/record.rs | 397 ++++++++++++++-------- drivers/gpu/drm/asahi/g15/board/words.rs | 5 +- drivers/gpu/drm/asahi/g15/hw.rs | 14 +- drivers/gpu/drm/asahi/g15/hw/t6031.rs | 24 +- drivers/gpu/drm/asahi/g15/initdata.rs | 63 +--- tools/asahi/g15-zero-words-tests.py | 91 +++-- 10 files changed, 398 insertions(+), 402 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/g15/board/blob.rs diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 2e8ca81a64505..647f8f37507d4 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -60,7 +60,6 @@ config DRM_ASAHI_G15 depends on DRM_APPLE=y || (DRM_APPLE=m && DRM_ASAHI=m) select APPLE_T6030_GPU_PMP_LINK select CRYPTO_LIB_SHA256 - select RUST_FW_LOADER_ABSTRACTIONS default n help Enable the native G15 backend for explicitly admitted M3 Pro/Max diff --git a/drivers/gpu/drm/asahi/g15/adt.rs b/drivers/gpu/drm/asahi/g15/adt.rs index aece4f083c0ad..83807d985866a 100644 --- a/drivers/gpu/drm/asahi/g15/adt.rs +++ b/drivers/gpu/drm/asahi/g15/adt.rs @@ -30,10 +30,13 @@ //! one cell per cluster). The same admission applies, without the DRAM bounds, and the four UAT //! regions may lie in System RAM: they are reserved-memory nodes without `no-map`, which the //! kernel maps, so the driver reaches them through its own mapping ([`super::regs::Mapping`]). +//! The GPU's sensor rows, which the node does not carry, come from the same ADT export as on +//! T6030: `/arm-io/aon-ptd` `ptd_data_table` ([`super::board::record::sensor_rows`]). use core::ops::Range; use kernel::{c_str, device, prelude::*}; +use super::board::record; use super::boot::glue; use super::hw; @@ -57,12 +60,13 @@ pub(crate) struct GpuAdt { pub(crate) perf_mhz: [u32; PERF_STATES], pub(crate) perf_mv: [u32; PERF_STATES], pub(crate) sram_mv: [u32; PERF_STATES], - /// Per-cluster core mV of each state, rounded up from the device tree's microvolts, how - /// many voltage cells the node has (1 on the ADT path), and the state's power in mW (0 on - /// the ADT path). + /// Per-cluster core mV of each state, rounded up from the device tree's microvolts, and how + /// many voltage cells the node has (1 on the ADT path). pub(crate) cluster_mv: [[u32; CLUSTERS]; PERF_STATES], pub(crate) cluster_cells: usize, - pub(crate) perf_mw: [u32; PERF_STATES], + /// The GPU's six sensor rows of the ADT's `aon-ptd` table (T6031; zero on the T6030 path, + /// whose templates carry their own). + pub(crate) sensor_rows: [[u8; 12]; record::SENSOR_ROW_COUNT], } /// Voltage cells the layout keeps per state. @@ -328,7 +332,7 @@ fn parse(dev: &device::Device, adt: &Adt<'_>) -> Result { sram_mv: [0; PERF_STATES], cluster_mv: [[0; CLUSTERS]; PERF_STATES], cluster_cells: 1, - perf_mw: [0; PERF_STATES], + sensor_rows: [[0; 12]; record::SENSOR_ROW_COUNT], }; collect_perf(dev, adt, sgx, &mut out)?; for i in 0..PERF_STATES { @@ -437,11 +441,11 @@ fn from_device_tree(dev: &device::Device) -> Result { sram_mv: [0; PERF_STATES], cluster_mv: [[0; CLUSTERS]; PERF_STATES], cluster_cells: 0, - perf_mw: [0; PERF_STATES], + sensor_rows: [[0; 12]; record::SENSOR_ROW_COUNT], }; // Operating points: MHz, the highest cluster voltage of each and the clusters' voltages - // (rounded up) and power for the hardware record; the SRAM voltage is not in the device - // tree. + // (rounded up) for the hardware record; the SRAM voltage is not in the device tree, and + // the powers are a boot-loader stand-in. let opps = node .parse_phandle(c_str!("operating-points-v2"), 0) .ok_or(EINVAL) @@ -450,7 +454,6 @@ fn from_device_tree(dev: &device::Device) -> Result { for opp in opps.children() { let hz: u64 = opp.get_property(c_str!("opp-hz"))?; let uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; - let uw: Option = opp.get_opt_property(c_str!("opp-microwatt"))?; let mv = uv.iter().max().copied().unwrap_or(0) / 1000; if uv.is_empty() || uv.len() > CLUSTERS || (count > 0 && uv.len() != out.cluster_cells) { dev_err!(dev, "GPU operating point {} has {} voltage cells\n", count, uv.len()); @@ -467,7 +470,6 @@ fn from_device_tree(dev: &device::Device) -> Result { out.cluster_mv[count][c] = v.div_ceil(1000); } out.cluster_cells = uv.len(); - out.perf_mw[count] = uw.unwrap_or(0) / 1000; } count += 1; } @@ -480,19 +482,44 @@ fn from_device_tree(dev: &device::Device) -> Result { ); return Err(EINVAL); } + out.sensor_rows = sensor_rows(dev)?; Ok(out) } +/// A private copy of the boot loader's ADT export. +fn copy() -> Result> { + let (start, size) = glue::adt_phram()?; + if size == 0 || size > 0x200000 || start & 0x3fff != 0 { + return Err(EINVAL); + } + super::identity::copy_physical(start, usize::try_from(size)?) +} + +/// The GPU's six sensor rows: `/arm-io/aon-ptd` `ptd_data_table` of the ADT export +/// ([`record::sensor_rows`]), which the device tree does not carry. +fn sensor_rows(dev: &device::Device) -> Result<[[u8; 12]; record::SENSOR_ROW_COUNT]> { + let copy = + copy().inspect_err(|_| dev_err!(dev, "no usable ADT export for the GPU sensor rows\n"))?; + let adt = Adt { data: © }; + let table = adt + .node_end(0, 0) + .and_then(|_| adt.child(0, "arm-io")) + .and_then(|armio| adt.child(armio, "aon-ptd")) + .and_then(|ptd| adt.prop(ptd, "ptd_data_table")) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "ADT has no aon-ptd ptd_data_table\n"))?; + record::sensor_rows(table).map_err(|why| { + dev_err!(dev, "ADT aon-ptd: {}\n", why); + EINVAL + }) +} + /// Reads and admits the GPU's regions, segments and performance states (read-only), from the /// source the SoC configuration names. pub(crate) fn read(dev: &device::Device, cfg: &super::Config) -> Result { if cfg.platform == hw::Platform::DeviceTree { return from_device_tree(dev); } - let (start, size) = glue::adt_phram()?; - if size == 0 || size > 0x200000 || start & 0x3fff != 0 { - return Err(EINVAL); - } - let copy = super::identity::copy_physical(start, usize::try_from(size)?)?; + let copy = copy()?; parse(dev, &Adt { data: © }) } diff --git a/drivers/gpu/drm/asahi/g15/board.rs b/drivers/gpu/drm/asahi/g15/board.rs index da0a7408575bf..f9682b2ead10c 100644 --- a/drivers/gpu/drm/asahi/g15/board.rs +++ b/drivers/gpu/drm/asahi/g15/board.rs @@ -10,11 +10,11 @@ //! against its hash at boot. //! //! A SoC whose board data comes from its device tree ([`super::hw::Platform::DeviceTree`]) gets -//! its hardware record from [`hardware_record`] ([`record`]) and its Globals and Power from -//! [`device_tree`]: the G15S templates with the GPU node's power words and the hardware -//! record's maximum power ([`words`]). +//! its hardware record from [`hardware_record`] ([`record`]: the GPU node, the ADT's sensor +//! rows and the SoC's reference words) and its Globals and Power from [`device_tree`]: the G15S +//! templates with the GPU node's power words and the hardware record's maximum power +//! ([`words`]). -pub(crate) mod blob; pub(crate) mod data; pub(crate) mod record; pub(crate) mod template; @@ -98,7 +98,6 @@ fn aux_opps( mhz, mv: [uv[0].div_ceil(1000), 0, 0, 0], cells: 1, - mw: 0, }; n += 1; } @@ -110,13 +109,13 @@ fn aux_opps( } /// The hardware record of a device-tree SoC ([`record`]): built from the GPU node's properties -/// and operating-point tables, the layout's IOMapping table and the ADT regions, with the -/// words the device tree does not carry copied from `file`, the machine's own record. +/// and operating-point tables, the layout's IOMapping table, the ADT regions and sensor rows, +/// and the SoC's `reference` words, which neither carries. pub(crate) fn hardware_record( dev: &device::Device, cfg: &super::Config, adt: &GpuAdt, - file: &[u8], + reference: &record::Reference, ) -> Result> { let layout = cfg.layout; let node = dev.of_node().ok_or(ENODEV)?; @@ -136,7 +135,6 @@ pub(crate) fn hardware_record( mhz: adt.perf_mhz[i], mv: adt.cluster_mv[i], cells: adt.cluster_cells, - mw: adt.perf_mw[i], }; } let mut cs = [record::Opp::default(); 16]; @@ -159,12 +157,6 @@ pub(crate) fn hardware_record( writable, }; } - let max_power_mw = u32::from_le_bytes( - file.get(record::MAX_POWER..record::MAX_POWER + 4) - .ok_or(EINVAL)? - .try_into() - .map_err(|_| EINVAL)?, - ); let inputs = record::Inputs { props, zones: &zones, @@ -177,15 +169,14 @@ pub(crate) fn hardware_record( usc_base: hw::client::COMPACT_WINDOW.start, user_end: hw::USER_RANGE.end, timestamps: va::TIMESTAMPS, - max_power_mw, + sensor_rows: adt.sensor_rows, + reference, }; let mut out = KVVec::from_elem(0u8, record::SIZE, GFP_KERNEL)?; - record::from_file(&mut out, file) - .and_then(|()| record::build(&mut out, &inputs)) - .map_err(|why| { - dev_err!(dev, "hardware record from the device tree: {}\n", why); - EINVAL - })?; + record::build(&mut out, &inputs).map_err(|why| { + dev_err!(dev, "hardware record from the device tree: {}\n", why); + EINVAL + })?; Ok(out) } diff --git a/drivers/gpu/drm/asahi/g15/board/blob.rs b/drivers/gpu/drm/asahi/g15/board/blob.rs deleted file mode 100644 index 04cff8bbaa3d9..0000000000000 --- a/drivers/gpu/drm/asahi/g15/board/blob.rs +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Board images from a firmware file. -//! -//! A SoC whose board images are not built-in templates names a firmware file -//! ([`super::super::hw::Board::Blob`]), built into the kernel with `CONFIG_EXTRA_FIRMWARE` or -//! loaded from `/lib/firmware`, and pins its SHA-256. The file holds the three images as the -//! firmware of that machine accepted them; the build then writes the pointers and policy words -//! it owns into them (TTBAT, IOMapping VAs, the timestamp arena, the idle policy). -//! -//! Layout, little-endian: the magic `T6031BRD`, u32 format version (1), u32 image count (3), -//! then per image `u32 kind, u32 size` with kinds 0 (HardwareData), 1 (Globals) and -//! 2 (Power), then the images' bytes in that order. Sizes must match the SoC's layout. - -use kernel::{device, firmware::Firmware, prelude::*}; - -use super::super::boot::glue; -use super::super::identity::Hex; - -const MAGIC: &[u8; 8] = b"T6031BRD"; -const VERSION: u32 = 1; -const IMAGES: usize = 3; - -/// The three images. -pub(crate) struct Images { - pub(crate) hardware: KVVec, - pub(crate) globals: KVVec, - pub(crate) power: KVVec, -} - -fn le32(b: &[u8], off: usize) -> Result { - Ok(u32::from_le_bytes( - b.get(off..off + 4) - .ok_or(EINVAL)? - .try_into() - .map_err(|_| EINVAL)?, - )) -} - -fn copy(b: &[u8], range: core::ops::Range) -> Result> { - let src = b.get(range).ok_or(EINVAL)?; - let mut out = KVVec::from_elem(0u8, src.len(), GFP_KERNEL)?; - out.copy_from_slice(src); - Ok(out) -} - -/// Requests `name`, verifies its digest against `sha256` and splits it into the images, each -/// of the size `sizes` gives for its kind. -pub(crate) fn load( - dev: &device::Device, - name: &CStr, - sha256: &[u8; 32], - sizes: [usize; IMAGES], -) -> Result { - let fw = Firmware::request(name, dev).inspect_err(|e| { - dev_err!(dev, "board images {}: not available ({:?})\n", name, e) - })?; - let data = fw.data(); - let digest = glue::sha256(data); - if digest != *sha256 { - dev_err!( - dev, - "board images {}: sha256 {} is not the pinned one\n", - name, - Hex(&digest) - ); - return Err(EINVAL); - } - if data.len() < 16 || &data[..8] != MAGIC || le32(data, 8)? != VERSION { - dev_err!(dev, "board images {}: unknown header\n", name); - return Err(EINVAL); - } - if le32(data, 12)? as usize != IMAGES { - dev_err!(dev, "board images {}: not {} images\n", name, IMAGES); - return Err(EINVAL); - } - let mut at = 16 + IMAGES * 8; - let mut images: [Option>; IMAGES] = [None, None, None]; - for i in 0..IMAGES { - let kind = le32(data, 16 + i * 8)? as usize; - let size = le32(data, 16 + i * 8 + 4)? as usize; - if kind != i || size != sizes[i] { - dev_err!( - dev, - "board images {}: image {} is kind {} of {} bytes, need {}\n", - name, - i, - kind, - size, - sizes[i] - ); - return Err(EINVAL); - } - images[i] = Some(copy(data, at..at + size)?); - at += size; - } - if at != data.len() { - dev_err!(dev, "board images {}: trailing bytes\n", name); - return Err(EINVAL); - } - let [hardware, globals, power] = images; - dev_info!( - dev, - "board images {}: sha256 {} accepted\n", - name, - Hex(&digest) - ); - Ok(Images { - hardware: hardware.ok_or(EINVAL)?, - globals: globals.ok_or(EINVAL)?, - power: power.ok_or(EINVAL)?, - }) -} diff --git a/drivers/gpu/drm/asahi/g15/board/record.rs b/drivers/gpu/drm/asahi/g15/board/record.rs index 959bc6a4ab0a2..119a692064391 100644 --- a/drivers/gpu/drm/asahi/g15/board/record.rs +++ b/drivers/gpu/drm/asahi/g15/board/record.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! The J516C hardware record (0xb480 bytes) from the device tree. +//! The J516C hardware record (0xb480 bytes) from the device tree and the ADT. //! //! The record is HardwareDataB (identity, IOMapping records, the DVFS tables) at 0, HardwareDataA //! (the power, performance and thermal controllers) at [`HWDATA_A`], and the G15C tail (sensor @@ -13,12 +13,14 @@ //! state cap and base state; //! - a layout constant of the 2419 firmware's G15C record: flags, the IOMapping flag words, //! the identity words, fixed factors (65536, 95, 100, 1.02, 0.8/0.2, 625, 11000, 125, 5); -//! - a word the device tree does not carry ([`FROM_FILE`]), copied from the machine's board file -//! by [`from_file`] and listed with its rule in the project notes: the maximum power -//! (HardwareDataA +0x6a4; every other maximum-power word derives from it), the MTR sensor -//! masks, the relative-power table (the device tree's operating-point powers are a boot-loader -//! stand-in) and the tail's six sensor rows; -//! - a word the file carries and the firmware was shown not to need ([`UNNEEDED`]): zero. +//! - the tail's six sensor rows: entries of the ADT's `aon-ptd` sensor table ([`sensor_rows`]); +//! - a word neither carries ([`REFERENCE_WORDS`]): the value of the J516C reference machine +//! ([`Reference`]), as the T6030 templates carry their reference machine's: the maximum +//! power (HardwareDataA +0x6a4; every other maximum-power word derives from it), the MTR +//! sensor masks and the relative-power table (the device tree's operating-point powers are a +//! boot-loader stand-in); +//! - a word the reference machine's record carries and the firmware was shown not to need +//! ([`UNNEEDED`]): zero. //! //! HardwareDataA has the G15S shape of the T6030 templates with insertions ([`a`]): 8 bytes //! before the fast-die controller (+0x8bc), 16-wide instead of 8-wide coefficient rows (+0x200 @@ -152,14 +154,13 @@ const SE_KP1: usize = 40; const SE_RESET: usize = 41; const SE_TARGET: usize = 42; -/// One operating point: MHz, the clusters' voltages in mV (rounded up from microvolts), how -/// many voltage cells the node had, and mW. +/// One operating point: MHz, the clusters' voltages in mV (rounded up from microvolts) and how +/// many voltage cells the node had. #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct Opp { pub(crate) mhz: u32, pub(crate) mv: [u32; CLUSTERS], pub(crate) cells: usize, - pub(crate) mw: u32, } impl Opp { @@ -198,40 +199,63 @@ pub(crate) struct Inputs<'a> { pub(crate) usc_base: u64, pub(crate) user_end: u64, pub(crate) timestamps: u64, - /// The maximum GPU power in mW ([`MAX_POWER`], a word the device tree does not carry). + /// The six sensor rows of the ADT's `aon-ptd` table ([`sensor_rows`]). + pub(crate) sensor_rows: [[u8; 12]; SENSOR_ROW_COUNT], + /// The words neither the device tree nor the ADT carries. + pub(crate) reference: &'a Reference, +} + +/// The record's words that neither the device tree nor the ADT carries, as one machine's +/// driver wrote them ([`REFERENCE_WORDS`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Reference { + /// The maximum GPU power in mW ([`MAX_POWER`]). pub(crate) max_power_mw: u32, + /// The relative powers of DVFS states 1..7, percent of the top state's (state 8 is 100). + pub(crate) relative_power: [u32; RELATIVE_STATES], + /// The MTR fast-die sensor mask (written twice) and alarm mask, as the u64 at each place. + /// The device tree's `apple,fast-die0-sensor-mask` is another value. + pub(crate) fast_die_mask: u64, + pub(crate) alarm_mask: u64, } +/// DVFS states with a relative-power word of their own. +pub(crate) const RELATIVE_STATES: usize = 7; + +/// The J516C reference machine's words (its 2419 firmware driver's values on that machine). +/// Another J516C gets them as a J514S gets the T6030 templates' machine's: the operating +/// points, voltages, gains and sensor rows are its own, these are the reference machine's. +pub(crate) const J516C: Reference = Reference { + max_power_mw: 116739, + relative_power: [11, 23, 33, 44, 55, 71, 90], + fast_die_mask: 0x802a002b_802a002b, + alarm_mask: 0x902a102b_802a002b, +}; + /// HardwareDataA's maximum-power word (= `fw::initdata::hwdata::MAX_POWER_A`). pub(crate) const MAX_POWER: usize = HWDATA_A + 0x6a4; +const RELATIVE_POWER: usize = 0x1060; +const FAST_DIE_MASK: usize = HWDATA_A + 0x8ac; +const FAST_DIE_MASK_2: usize = HWDATA_A + 0x1490; +const ALARM_MASK: usize = HWDATA_A + 0x24a8; -/// Words the device tree does not carry, copied from the board file. Record offsets. +/// The record words [`Reference`] supplies. Record offsets. +#[cfg(test)] #[rustfmt::skip] -pub(crate) const FROM_FILE: [usize; 32] = [ - // The relative powers of states 1..7 (state 8 is 100): the device tree's operating-point - // powers are a boot-loader stand-in. +pub(crate) const REFERENCE_WORDS: [usize; 14] = [ 0x1060, 0x1064, 0x1068, 0x106c, 0x1070, 0x1074, 0x1078, - // HardwareDataA: the maximum power, the MTR fast-die masks (+0x8ac, +0x1490) and the alarm - // mask (+0x24a8); the device tree's apple,fast-die0-sensor-mask is another value. MAX_POWER, - HWDATA_A + 0x8ac, HWDATA_A + 0x8b0, - HWDATA_A + 0x1490, HWDATA_A + 0x1494, - HWDATA_A + 0x24a8, HWDATA_A + 0x24ac, - // The tail's six sensor rows (key, id, kind; 12 bytes each; the boot loader does not export - // the aon-ptd table). - HWDATA_A + 0x6c58, HWDATA_A + 0x6c5c, HWDATA_A + 0x6c60, - HWDATA_A + 0x6c64, HWDATA_A + 0x6c68, HWDATA_A + 0x6c6c, - HWDATA_A + 0x6c70, HWDATA_A + 0x6c74, HWDATA_A + 0x6c78, - HWDATA_A + 0x6c7c, HWDATA_A + 0x6c80, HWDATA_A + 0x6c84, - HWDATA_A + 0x6c88, HWDATA_A + 0x6c8c, HWDATA_A + 0x6c90, - HWDATA_A + 0x6c94, HWDATA_A + 0x6c98, HWDATA_A + 0x6c9c, + FAST_DIE_MASK, FAST_DIE_MASK + 4, + FAST_DIE_MASK_2, FAST_DIE_MASK_2 + 4, + ALARM_MASK, ALARM_MASK + 4, ]; -/// Words the board file carries that the record leaves zero: the firmware ran, passed every -/// check and reached its top state without them (one boot each). The cross-check accepts the -/// file's value there. 27 words in the HardwareDataB +0x38..+0x638 block and two all-ones words +/// Words the reference machine's record carries that this one leaves zero: the firmware ran, +/// passed every check and reached its top state without them (one boot each). 27 words in +/// the HardwareDataB +0x38..+0x638 block and two all-ones words /// after it (zero in the G15S templates); the tail's thirteen readings (a single at /// HardwareDataA +0x6d34, twelve singles after +0x6d80). +#[cfg(test)] #[rustfmt::skip] pub(crate) const UNNEEDED: [usize; 42] = [ 0xe0, 0xe8, 0xf4, 0xf8, 0xfc, 0x100, 0x104, 0x108, 0x10c, 0x2d8, 0x2dc, 0x2e0, 0x2e4, @@ -247,7 +271,7 @@ pub(crate) const UNNEEDED: [usize; 42] = [ /// rows whose kind byte is `g`, their presence mask, then the `r` and `f` rows' keys with a /// count of one each. const SENSOR_ROWS: usize = HWDATA_A + 0x6c58; -const SENSOR_ROW_COUNT: usize = 6; +pub(crate) const SENSOR_ROW_COUNT: usize = 6; const SENSOR_G_KEYS: usize = HWDATA_A + 0x6cdc; const SENSOR_G_MASK: usize = HWDATA_A + 0x6cfc; const SENSOR_R_KEY: usize = HWDATA_A + 0x6d04; @@ -492,15 +516,13 @@ fn hwdata_b(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { put32(out, 0x10dc + 4 * k, t.primary[k] as u32)?; put32(out, 0x111c + 4 * k, t.secondary[k] as u32)?; if k > 0 { - // Relative power (a FROM_FILE word on the J516C) and relative boost frequency. - if top.mw != 0 { - put32( - out, - 0x105c + 4 * k, - u32::try_from(100 * u64::from(ps.mw) / u64::from(top.mw)) - .map_err(|_| "a relative power does not fit its u32 word")?, - )?; - } + // Relative power (the reference machine's) and relative boost frequency. + let relative = match i.reference.relative_power.get(k - 1) { + Some(&r) if k < t.top() => r, + None if k == t.top() => 100, + _ => return Err("the reference relative powers are for another state count"), + }; + put32(out, RELATIVE_POWER - 4 + 4 * k, relative)?; if ps.mhz > base.mhz { let span = u64::from(top.mhz - base.mhz); put32( @@ -565,7 +587,7 @@ fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { |gain: usize| f32_mul(p[gain], period_s).ok_or("a gain is not a finite normal single"); let max_scaled = 100 * t.top() as u32; let base_scaled = mul(100, p[BASE_PSTATE])?; - let max_power = i.max_power_mw; + let max_power = i.reference.max_power_mw; let max_power_f = f32_from_u32(max_power); let mut w = |off: usize, v: u32| put32(out, a(off), v); w(0x4, p[AIC_CLKS])?; @@ -644,7 +666,7 @@ fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { w(0x864, max_scaled)?; w(0x86c, F_0_8)?; w(0x870, F_0_2)?; - // Fast-die controller (its sensor mask is a FROM_FILE word). + // Fast-die controller (its sensor mask follows below). w(0x8bc, 4)?; w(0x8d0, F_1)?; w(0x8d8, per_period(FD_KI)?)?; @@ -737,15 +759,46 @@ fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { w(SE + 0x11c, F_65536)?; w(SE + 0x12c, 40)?; w(SE + 0x130, max_scaled)?; - // Past the second sensor mask (FROM_FILE), the alarm mask (FROM_FILE) and the end flag. + // Past the second sensor mask, the alarm mask and the end flag. w(0x1290, 125)?; w(0x1294, 1)?; w(0x1aa0, 1)?; w(0x3aa4, F_5)?; + // The MTR sensor masks: the fast-die mask at both places, then the alarm mask. + put64(out, FAST_DIE_MASK, i.reference.fast_die_mask)?; + put64(out, FAST_DIE_MASK_2, i.reference.fast_die_mask)?; + put64(out, ALARM_MASK, i.reference.alarm_mask)?; Ok(()) } -/// The tail's sensor keys from the six rows (which [`from_file`] fills). +/// The six sensor rows of the ADT's `aon-ptd` `ptd_data_table`: 12-byte entries of a +/// little-endian four-character key (`RT0g` is stored as `"RT0g"`) and two words. The rows are +/// the `RT` entries of kind `g` (four, in table order), then the one of kind `r` and the one +/// of kind `f`; any other count is refused rather than guessed. +pub(crate) fn sensor_rows(table: &[u8]) -> R<[[u8; 12]; SENSOR_ROW_COUNT]> { + if table.is_empty() || table.len() % 12 != 0 { + return Err("the aon-ptd sensor table is not whole 12-byte entries"); + } + let mut rows = [[0u8; 12]; SENSOR_ROW_COUNT]; + let mut n = 0; + for (kind, wanted) in [(b'g', 4), (b'r', 1), (b'f', 1)] { + let mut found = 0; + for e in table.chunks_exact(12) { + if e[..2] == *b"RT" && e[3] == kind { + let row = rows.get_mut(n).ok_or("too many GPU sensor rows in aon-ptd")?; + row.copy_from_slice(e); + n += 1; + found += 1; + } + } + if found != wanted { + return Err("aon-ptd does not hold four g, one r and one f GPU sensor row"); + } + } + Ok(rows) +} + +/// The tail's sensor keys from the six rows. fn tail(out: &mut [u8]) -> R<()> { let mut g = 0usize; for row in 0..SENSOR_ROW_COUNT { @@ -775,16 +828,10 @@ fn tail(out: &mut [u8]) -> R<()> { Ok(()) } -/// Copies the [`FROM_FILE`] words of `file` (the board file's record) into `out`. -pub(crate) fn from_file(out: &mut [u8], file: &[u8]) -> R<()> { - for &at in FROM_FILE.iter() { - put32(out, at, get32(file, at)?)?; - } - Ok(()) -} - -/// The first word at which the built record differs from the file's, except the [`UNNEEDED`] -/// words, where the file's value is accepted as long as the built word is zero. +/// The first word at which a built record differs from a machine's own record (a host test's +/// fixture), except the [`UNNEEDED`] words, where the machine's value is accepted as long as +/// the built word is zero. +#[cfg(test)] pub(crate) fn cross_check(built: &[u8], file: &[u8]) -> Option<(usize, u32, u32)> { let word = |s: &[u8], at: usize| -> u32 { let mut w = [0u8; 4]; @@ -799,23 +846,20 @@ pub(crate) fn cross_check(built: &[u8], file: &[u8]) -> Option<(usize, u32, u32) .find(|&(at, b, f)| b != f && !(b == 0 && UNNEEDED.contains(&at))) } -/// Builds the record into `out` (zeroed first) from the device-tree inputs; the [`FROM_FILE`] -/// words must have been copied in first ([`from_file`]) or are left as they are. +/// Builds the record into `out` (zeroed first) from the inputs. pub(crate) fn build(out: &mut [u8], i: &Inputs<'_>) -> R<()> { if out.len() != SIZE { return Err("record of another size"); } - let mut keep = [0u32; FROM_FILE.len()]; - for (k, &at) in FROM_FILE.iter().enumerate() { - keep[k] = get32(out, at)?; - } out.fill(0); let t = tables(i.main)?; hwdata_b(out, i, &t)?; hwdata_a(out, i, &t)?; - // The kept words win over what the rules put there (the relative powers, the masks). - for (k, &at) in FROM_FILE.iter().enumerate() { - put32(out, at, keep[k])?; + for (k, row) in i.sensor_rows.iter().enumerate() { + let at = SENSOR_ROWS + 12 * k; + out.get_mut(at..at + 12) + .ok_or("record too short")? + .copy_from_slice(row); } tail(out) } @@ -828,18 +872,13 @@ mod tests { get32(b, o).unwrap() } - fn opp(mhz: u32, mv: [u32; 4], mw: u32) -> Opp { - Opp { - mhz, - mv, - cells: 4, - mw, - } + fn opp(mhz: u32, mv: [u32; 4]) -> Opp { + Opp { mhz, mv, cells: 4 } } /// Operating points shaped like the J516C's: eight voltages, the upper five with two states. fn main_opps() -> Vec { - let mut v = vec![opp(0, [125; 4], 0)]; + let mut v = vec![opp(0, [125; 4])]; let groups: [(u32, [u32; 4], &[u32]); 8] = [ (605, [605, 605, 615, 615], &[338]), (680, [680, 680, 665, 665], &[618]), @@ -852,7 +891,7 @@ mod tests { ]; for (_, mv, mhz) in groups { for &m in mhz { - v.push(opp(m, mv, m * 40)); + v.push(opp(m, mv)); } } v @@ -888,6 +927,27 @@ mod tests { p } + /// Arbitrary reference words, distinct from the J516C's. + const TEST_REFERENCE: Reference = Reference { + max_power_mw: 54321, + relative_power: [0x1000, 0x1001, 0x1002, 0x1003, 0x1004, 0x1005, 0x1006], + fast_die_mask: 0x1111_2222_3333_4444, + alarm_mask: 0x5555_6666_7777_8888, + }; + + /// A sensor row: the key's characters in storage order, then two words. + fn row(key: &[u8; 4], id: u32, kind: u32) -> [u8; 12] { + let mut r = [0u8; 12]; + r[..4].copy_from_slice(key); + r[4..8].copy_from_slice(&id.to_le_bytes()); + r[8..].copy_from_slice(&kind.to_le_bytes()); + r + } + + fn rows() -> [[u8; 12]; SENSOR_ROW_COUNT] { + [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"].map(|k| row(k, 0x10001, 0x1401)) + } + fn inputs<'a>(main: &'a [Opp], cs: &'a [Opp], slots: &'a [IoSlot]) -> Inputs<'a> { Inputs { props: props(), @@ -901,7 +961,8 @@ mod tests { usc_base: 0x10_0000_0000, user_end: 0x2ff_ffff_8000, timestamps: 0xfffffc2ffc000000, - max_power_mw: 54321, + sensor_rows: rows(), + reference: &TEST_REFERENCE, } } @@ -927,9 +988,12 @@ mod tests { #[test] fn unneeded() { - // No UNNEEDED word is a FROM_FILE word; the cross-check accepts the file's value there - // only against a zero built word, and nothing else. - assert!(UNNEEDED.iter().all(|at| !FROM_FILE.contains(at))); + // No UNNEEDED word is a reference or sensor-row word; the cross-check accepts the + // file's value there only against a zero built word, and nothing else. + let rows = SENSOR_ROWS..SENSOR_ROWS + 12 * SENSOR_ROW_COUNT; + assert!(UNNEEDED + .iter() + .all(|at| !REFERENCE_WORDS.contains(at) && !rows.contains(at))); assert!(UNNEEDED.iter().all(|&at| at % 4 == 0 && at + 4 <= SIZE)); let mut file = vec![0u8; SIZE]; for (k, &at) in UNNEEDED.iter().enumerate() { @@ -957,9 +1021,10 @@ mod tests { assert_eq!(a(0x1aa0), HWDATA_A + 0x24b8); assert_eq!(a(0x3aa4), HWDATA_A + 0x64bc); assert_eq!(MAX_POWER, a(0x6a4)); - // FROM_FILE is sorted, unique and inside the record. - assert!(FROM_FILE.windows(2).all(|w| w[0] < w[1])); - assert!(FROM_FILE.iter().all(|&o| o % 4 == 0 && o + 4 <= SIZE)); + // REFERENCE_WORDS is sorted, unique and inside the record. + assert!(REFERENCE_WORDS.windows(2).all(|w| w[0] < w[1])); + assert!(REFERENCE_WORDS.iter().all(|&o| o % 4 == 0 && o + 4 <= SIZE)); + assert!(SENSOR_ROWS + 12 * SENSOR_ROW_COUNT <= SENSOR_G_KEYS); } #[test] @@ -995,7 +1060,6 @@ mod tests { mhz, mv: [mv, 0, 0, 0], cells: 1, - mw: 0, }) .collect(); let slots = [ @@ -1017,20 +1081,23 @@ mod tests { }, ]; let i = inputs(&main, &cs, &slots); - let mut r = vec![0u8; SIZE]; - // FROM_FILE words survive the build; the sensor rows feed the tail. - for (k, &at) in FROM_FILE.iter().enumerate() { - put32(&mut r, at, 0x1000 + k as u32).unwrap(); - } - // The file's maximum power is what the input repeats. - put32(&mut r, MAX_POWER, 54321).unwrap(); - let rows = [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"]; - for (k, row) in rows.iter().enumerate() { - put32(&mut r, SENSOR_ROWS + 12 * k, u32::from_le_bytes(**row)).unwrap(); - } + // Whatever the buffer held before is not part of the record. + let mut r = vec![0x5au8; SIZE]; build(&mut r, &i).unwrap(); let good = r.clone(); - assert_eq!(get(&r, 0x1060), 0x1000); + // The reference words: relative powers of states 1..7, the maximum power, the masks. + for k in 0..RELATIVE_STATES { + assert_eq!(get(&r, 0x1060 + 4 * k), 0x1000 + k as u32); + } + assert_eq!(get(&r, FAST_DIE_MASK), 0x3333_4444); + assert_eq!(get(&r, FAST_DIE_MASK + 4), 0x1111_2222); + assert_eq!(get(&r, FAST_DIE_MASK_2), 0x3333_4444); + assert_eq!(get(&r, FAST_DIE_MASK_2 + 4), 0x1111_2222); + assert_eq!(get(&r, ALARM_MASK), 0x7777_8888); + assert_eq!(get(&r, ALARM_MASK + 4), 0x5555_6666); + // The sensor rows as given. + assert_eq!(&r[SENSOR_ROWS..SENSOR_ROWS + 12], &rows()[0]); + assert_eq!(&r[SENSOR_ROWS + 60..SENSOR_ROWS + 72], &rows()[5]); assert_eq!(get(&r, 0x30f0), 0); // Identity, IOMapping records, flags. assert_eq!(get(&r, 0xa28), 0x6031); @@ -1099,8 +1166,15 @@ mod tests { let mut j = inputs(&main, &cs, &slots); j.empty_writable = &[0]; assert!(build(&mut r, &j).is_err()); + let zero = Reference { + max_power_mw: 0, + ..TEST_REFERENCE + }; + let mut j = inputs(&main, &cs, &slots); + j.reference = &zero; + assert!(build(&mut r, &j).is_err()); let mut j = inputs(&main, &cs, &slots); - j.max_power_mw = 0; + j.sensor_rows[0] = row(b"RT0x", 1, 1); assert!(build(&mut r, &j).is_err()); let mut j = inputs(&main, &cs, &slots); j.zones = &[100, 101, 1]; @@ -1127,34 +1201,49 @@ mod tests { let mut j = inputs(&main, &cs, &slots); j.zones = &[5000, 200, 0x4000_0000]; assert_eq!(refused(&j), Err(OVERFLOW)); - let mut skewed = main.clone(); - skewed[1].mw = u32::MAX; - skewed[13].mw = 1; + // The seven relative powers fit eight states above state 0 and nothing else. + let fewer = &main[..PERF_STATES - 2]; + assert_eq!(tables(fewer).unwrap().top(), 7); assert_eq!( - refused(&inputs(&skewed, &cs, &slots)), - Err("a relative power does not fit its u32 word") + refused(&inputs(fewer, &cs, &slots)), + Err("the reference relative powers are for another state count") ); - // The admission cross-check as initdata::blob() composes it: the FROM_FILE words from - // the file, the rest built, then the first differing word against the file. A file the - // inputs reproduce passes; a rule word the file holds differently is found at its - // offset; a FROM_FILE word follows the file and is never a difference. - let first_difference = super::super::words::first_difference; - let checked = |file: &[u8]| { - let mut built = vec![0u8; SIZE]; - from_file(&mut built, file).unwrap(); - build(&mut built, &i).unwrap(); - (first_difference(&built, file), built) - }; - assert_eq!(checked(&good).0, None); - let mut file = good.clone(); - put32(&mut file, a(0x4c), 801).unwrap(); - assert_eq!(checked(&file).0, Some((a(0x4c), 800, 801))); - let mut file = good.clone(); - assert!(FROM_FILE.contains(&0x1064)); - put32(&mut file, 0x1064, get(&good, 0x1064) ^ 0x5a).unwrap(); - let (difference, built) = checked(&file); - assert_eq!(difference, None); - assert_eq!(get(&built, 0x1064), get(&good, 0x1064) ^ 0x5a); + } + + #[test] + fn sensor_rows_from_the_ptd_table() { + // A table shaped like the J516C's: other sensors around the GPU's, the f row first. + let mut table = Vec::new(); + for r in [ + row(b"YD0C", 0x5e0001, 0x1040405), + row(b"RT0a", 0x6b0001, 0x13), + row(b"RT0f", 0xcb0001, 0x1601), + row(b"RE0g", 0xaa0001, 0xff01), + row(b"RT0g", 0xab0001, 0x1401), + row(b"RT1g", 0x8d0001, 0x1401), + row(b"RT2g", 0x8b0001, 0x1401), + row(b"RT3g", 0xad0001, 0x1401), + row(b"RT0p", 0x6d0001, 0x13), + row(b"RT0r", 0xeb0001, 0x1401), + row(b"XEXE", 0x3c0001, 0xff00), + ] { + table.extend_from_slice(&r); + } + let got = sensor_rows(&table).unwrap(); + let keys: Vec<&[u8]> = got.iter().map(|r| &r[..4]).collect(); + assert_eq!(keys, [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"]); + assert_eq!(got[5], row(b"RT0f", 0xcb0001, 0x1601)); + assert_eq!(got[4], row(b"RT0r", 0xeb0001, 0x1401)); + // Refused: no table, a partial entry, a missing row, a fifth g row, a second f row. + assert!(sensor_rows(&[]).is_err()); + assert!(sensor_rows(&table[..table.len() - 1]).is_err()); + assert!(sensor_rows(&table[12..table.len() - 12]).is_ok()); + assert!(sensor_rows(&table[..9 * 12]).is_err()); + for extra in [row(b"RT4g", 1, 1), row(b"RT1f", 1, 1)] { + let mut t = table.clone(); + t.extend_from_slice(&extra); + assert!(sensor_rows(&t).is_err()); + } } // --- The fixture: a saved copy of a J516C's device tree and its private board file. --- @@ -1236,14 +1325,12 @@ mod tests { for (c, m) in mv.iter_mut().enumerate().take(cells) { *m = be32(uv, 4 * c).div_ceil(1000); } - let mw = prop(props, "opp-microwatt").map_or(0, |w| be32(w, 0) / 1000); ( p.clone(), Opp { mhz: (hz / 1_000_000) as u32, mv, cells, - mw, }, ) }) @@ -1262,16 +1349,8 @@ mod tests { mhz: 24, mv: [635, 0, 0, 0], cells: 1, - mw: 0, }]; let mut seed = vec![0u8; SIZE]; - put32(&mut seed, MAX_POWER, 54321).unwrap(); - for (k, key) in [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"] - .iter() - .enumerate() - { - put32(&mut seed, SENSOR_ROWS + 12 * k, u32::from_le_bytes(**key)).unwrap(); - } assert_eq!(build(&mut seed, &inputs(&main, &cs, &[])), Ok(())); for prop in 0..PROPS.len() { for value in [ @@ -1289,16 +1368,59 @@ mod tests { } } - /// Builder on the saved device tree + the file's FROM_FILE words == the file's hardware - /// record, byte for byte. Needs `G15_J516C_FIXTURES=` with `j516c.dtb` and - /// `t6031-j516c-board.bin` (off the tree): ignored by default, `--ignored` runs it. + /// A property of an ADT node found by its path of `name`s from the root: nodes of u32 + /// property and child counts, properties of a 32-byte name, a u32 size (top bit a flag) + /// and the value padded to 4 bytes. + fn adt_prop<'a>(adt: &'a [u8], path: &[&str], name: &str) -> Option<&'a [u8]> { + fn le(b: &[u8], o: usize) -> usize { + u32::from_le_bytes(b[o..o + 4].try_into().unwrap()) as usize + } + // The node at `at`: its properties, its children's offsets and the offset past it. + fn node(adt: &[u8], at: usize) -> (Vec<(&[u8], &[u8])>, Vec, usize) { + let (np, nc) = (le(adt, at), le(adt, at + 4)); + let mut o = at + 8; + let mut props = Vec::new(); + for _ in 0..np { + let n = &adt[o..o + 32]; + let n = &n[..n.iter().position(|&b| b == 0).unwrap_or(32)]; + let size = le(adt, o + 32) & 0x7fff_ffff; + props.push((n, &adt[o + 36..o + 36 + size])); + o += 36 + size.next_multiple_of(4); + } + let mut children = Vec::new(); + for _ in 0..nc { + children.push(o); + o = node(adt, o).2; + } + (props, children, o) + } + let mut at = 0; + for want in path { + let (_, children, _) = node(adt, at); + let named = |c: usize| { + let (props, _, _) = node(adt, c); + props.iter().any(|(n, v)| { + *n == b"name" && v.split(|&b| b == 0).next() == Some(want.as_bytes()) + }) + }; + at = *children.iter().find(|&&c| named(c))?; + } + node(adt, at).0.into_iter().find(|(n, _)| *n == name.as_bytes()).map(|(_, v)| v) + } + + /// Builder on the saved device tree and ADT with the J516C reference words == the J516C + /// reference machine's own hardware record, byte for byte except the [`UNNEEDED`] words. + /// Needs `G15_J516C_FIXTURES=` with `j516c.dtb` (`/sys/firmware/fdt`), `adt.bin` (the + /// boot loader's ADT) and `t6031-j516c-board.bin` (that machine's record), all off the + /// tree: ignored by default, `--ignored` runs it. #[test] #[ignore = "needs the off-tree J516C fixtures (G15_J516C_FIXTURES)"] fn j516c_fixture() { let dir = std::env::var_os("G15_J516C_FIXTURES") - .expect("G15_J516C_FIXTURES names the directory with j516c.dtb and the board file"); + .expect("G15_J516C_FIXTURES names the directory with j516c.dtb, adt.bin, the record"); let dir = std::path::Path::new(&dir); let dtb = std::fs::read(dir.join("j516c.dtb")).unwrap(); + let adt = std::fs::read(dir.join("adt.bin")).unwrap(); let blob = std::fs::read(dir.join("t6031-j516c-board.bin")).unwrap(); assert_eq!(&blob[..8], b"T6031BRD"); let at = 16 + 3 * 8; @@ -1389,10 +1511,13 @@ mod tests { usc_base: 0x10_0000_0000, user_end: 0x2ff_ffff_8000, timestamps: 0xfffffc2ffc000000, - max_power_mw: get(file, MAX_POWER), + sensor_rows: sensor_rows( + adt_prop(&adt, &["arm-io", "aon-ptd"], "ptd_data_table").expect("aon-ptd table"), + ) + .unwrap(), + reference: &J516C, }; let mut built = vec![0u8; SIZE]; - from_file(&mut built, file).unwrap(); build(&mut built, &i).unwrap(); assert_eq!(cross_check(&built, file), None, "(offset, built, file)"); for &at in UNNEEDED.iter() { diff --git a/drivers/gpu/drm/asahi/g15/board/words.rs b/drivers/gpu/drm/asahi/g15/board/words.rs index 4c5082465d428..f879d9b671672 100644 --- a/drivers/gpu/drm/asahi/g15/board/words.rs +++ b/drivers/gpu/drm/asahi/g15/board/words.rs @@ -235,6 +235,7 @@ pub(crate) fn power( /// The first word (4-byte aligned offset) at which `a` and `b` differ, with both values; a /// missing word reads 0. `None` when they are equal. +#[cfg(test)] pub(crate) fn first_difference(a: &[u8], b: &[u8]) -> Option<(usize, u32, u32)> { let word = |s: &[u8], at: usize| -> u32 { let mut w = [0u8; 4]; @@ -584,7 +585,7 @@ mod tests { /// Builder on the saved device tree == the private file's Globals and Power, byte for byte. /// Needs `G15_J516C_FIXTURES=` with `j516c.dtb` (`/sys/firmware/fdt` of the machine) - /// and `t6031-j516c-board.bin` (its board file); both stay off the tree, so the test is + /// and `t6031-j516c-board.bin` (its own images); both stay off the tree, so the test is /// ignored by default (`--ignored` runs it) and fails, rather than passes, without them. #[test] #[ignore = "needs the off-tree J516C fixtures (G15_J516C_FIXTURES)"] @@ -595,7 +596,7 @@ mod tests { let dtb = std::fs::read(dir.join("j516c.dtb")).unwrap(); let blob = std::fs::read(dir.join("t6031-j516c-board.bin")).unwrap(); - // The board file: magic, version, count, 3 x (kind, size), images (board/blob.rs). + // The machine's images: magic, version, count, 3 x (kind, size), then the images. assert_eq!(&blob[..8], b"T6031BRD"); assert_eq!(get32(&blob, 8), 1); assert_eq!(get32(&blob, 12), 3); diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index c7f21a9110cee..4f2e9f6d3de1f 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -46,15 +46,11 @@ pub(crate) enum Platform { pub(crate) enum Board { /// The hash-checked T6030 templates ([`super::board`]). Templates, - /// The three images built from the device tree ([`super::board::record`], - /// [`super::board::device_tree`]) and cross-checked against a firmware file holding the - /// machine's own ([`super::board::blob`], pinned by its SHA-256), which also supplies the - /// record's words the device tree does not carry ([`super::board::record::FROM_FILE`]); a - /// difference refuses admission, before any firmware is touched. - Blob { - name: &'static kernel::str::CStr, - sha256: [u8; 32], - }, + /// The three images built from the device tree and the ADT's sensor rows + /// ([`super::board::record`], [`super::board::device_tree`]), with the SoC's reference + /// words for what neither carries ([`super::board::record::Reference`]). A value the + /// builder refuses stops admission, before any firmware is touched. + DeviceTree(&'static super::board::record::Reference), /// No board data: admission refuses the SoC before anything firmware-visible. None, } diff --git a/drivers/gpu/drm/asahi/g15/hw/t6031.rs b/drivers/gpu/drm/asahi/g15/hw/t6031.rs index 4af32f6068cd7..8bac33989db64 100644 --- a/drivers/gpu/drm/asahi/g15/hw/t6031.rs +++ b/drivers/gpu/drm/asahi/g15/hw/t6031.rs @@ -8,9 +8,8 @@ //! device tree node the boot loader fills, not an ADT export), the GPU power (its power domain; //! there is no PMP vote), the initdata root version, a longer HardwareData record (0xb480 //! bytes, its fourth view at +0xb400) placed aligned and ASC-cached, the IOMapping table (17 -//! slots at the T6031 register blocks) and the board images, built from the device tree and -//! checked against this machine's firmware file, which supplies the record's non-device-tree -//! words. Every +//! slots at the T6031 register blocks) and the board images, built from the device tree, the +//! ADT's sensor rows and the J516C reference words. Every //! value was measured on a J516C: the SGX identification //! registers, the loaded firmware segment, the device tree the boot loader hands over and the //! initdata of that machine. @@ -117,18 +116,6 @@ pub(crate) const LAYOUT: Layout = Layout { record_empty_writable: &RECORD_EMPTY_WRITABLE, }; -/// The J516C board file (`asahi/t6031-j516c-board.bin`, built into the kernel): the hardware -/// record, Globals and the power block that this machine's firmware accepted, as the earlier -/// J516C driver uploaded them (its device-tree-built records with the per-board words of this -/// machine applied). Kept out of the tree; pinned by its SHA-256. The three images are built -/// from the device tree and must match the file's; the file supplies the record's words the -/// device tree does not carry (`board::record::FROM_FILE`). -const BOARD_SHA256: [u8; 32] = [ - 0x4e, 0x03, 0xf2, 0x9f, 0xa4, 0x47, 0xdc, 0xeb, 0xfd, 0x4d, 0x1c, 0x22, 0x5a, 0x64, 0x7b, - 0xfa, 0x31, 0xb3, 0x63, 0x45, 0x8f, 0x28, 0x33, 0x82, 0x3a, 0x5c, 0xa8, 0xa1, 0x0e, 0xdf, - 0xf3, 0xfc, -]; - pub(crate) const CONFIG: Config = Config { name: "T6031 G15C", chip_id: 0x6031, @@ -172,15 +159,12 @@ pub(crate) const CONFIG: Config = Config { ], }, ], - // The pinned board images and measured layout belong to the J516C, not + // The reference words and measured layout belong to the J516C, not // every T6031 board (in particular, not the 14-inch J514C). machine: kernel::c_str!("apple,j516c"), platform: super::Platform::DeviceTree, layout: &LAYOUT, - board: super::Board::Blob { - name: kernel::c_str!("asahi/t6031-j516c-board.bin"), - sha256: BOARD_SHA256, - }, + board: super::Board::DeviceTree(&super::super::board::record::J516C), // Four clusters: the saved-state pointers at +0x2980 of the preemption buffer (0xd80 plus // 0x700 per cluster), UMA PTC 0x6000400020, DUPM 0x3c000000140 (32 N | 96 N << 32 for // N = 10 cores per cluster) and 0x1a0e9 = 0, with the saved-state pointers and the diff --git a/drivers/gpu/drm/asahi/g15/initdata.rs b/drivers/gpu/drm/asahi/g15/initdata.rs index f3f5232e92a28..c085d455f7a40 100644 --- a/drivers/gpu/drm/asahi/g15/initdata.rs +++ b/drivers/gpu/drm/asahi/g15/initdata.rs @@ -160,58 +160,23 @@ fn templates( /// HardwareDataA word holding the system counter at initdata creation. const HWDATA_A_INIT_TIMESTAMP: usize = 0x424c; -/// Board images of a SoC with a firmware file. The hardware record ([`board::hardware_record`]), -/// Globals and Power ([`board::device_tree`]) are built from the device tree; the file, this -/// machine's own, supplies the record's words the device tree does not carry and is otherwise -/// a cross-check: admission ([`prepare_board`], before any firmware is touched) is refused at -/// the first differing word. Returns the images and the top operating-point frequency in kHz. -fn blob( +/// Board images of a device-tree SoC: the hardware record ([`board::hardware_record`]) from the +/// GPU node, the ADT's sensor rows and the SoC's `reference` words, Globals and Power +/// ([`board::device_tree`]) from the GPU node and that record. Returns the images and the top +/// operating-point frequency in kHz. +fn device_tree_board( dev: &device::Device, cfg: &Config, adt: &GpuAdt, - name: &CStr, - sha256: &[u8; 32], + reference: &board::record::Reference, ) -> Result<(KVVec, KVVec, KVVec, u32)> { - let layout = cfg.layout; - let images = board::blob::load( - dev, - name, - sha256, - [layout.hwdata_size, globals::SIZE, power::SIZE], - )?; - let hw_img = board::hardware_record(dev, cfg, adt, &images.hardware)?; + let hw_img = board::hardware_record(dev, cfg, adt, reference)?; let (glob, pwr, max_power_mw) = board::device_tree(dev, &hw_img)?; - for (what, built, file) in [ - ("hardware record", &hw_img, &images.hardware), - ("Globals", &glob, &images.globals), - ("Power", &pwr, &images.power), - ] { - let diff = if what == "hardware record" { - board::record::cross_check(built, file) - } else { - board::words::first_difference(built, file) - }; - if let Some((at, b, f)) = diff { - dev_err!( - dev, - "board data: device-tree {} differs from {} at +{:#x}: built {:#x}, file {:#x}\n", - what, - name, - at, - b, - f - ); - return Err(EINVAL); - } - } let max_mhz = adt.perf_mhz.iter().copied().max().unwrap_or(0); dev_info!( dev, - "board data: {} images from the device tree match {} ({} record words from the file, {} file words left zero), {} mW maximum power, {} MHz top operating point\n", + "board data: {} images from the device tree and the ADT sensor rows, reference maximum power {} mW and MTR masks, {} MHz top operating point\n", cfg.name, - name, - board::record::FROM_FILE.len(), - board::record::UNNEEDED.len(), max_power_mw, max_mhz ); @@ -234,7 +199,7 @@ pub(crate) fn prepare_board( ) -> Result { match cfg.board { hw::Board::Templates => templates(dev, adt), - hw::Board::Blob { name, sha256 } => blob(dev, cfg, adt, name, &sha256), + hw::Board::DeviceTree(reference) => device_tree_board(dev, cfg, adt, reference), hw::Board::None => Err(ENODEV), } } @@ -296,7 +261,7 @@ pub(crate) fn build( // Board images, verified, then the owned replacements. let (mut hw_img, mut glob, pwr, max_khz) = board_images; - if matches!(cfg.board, hw::Board::Blob { .. }) { + if matches!(cfg.board, hw::Board::DeviceTree(_)) { // This timestamp belongs to initdata creation, not board-data preflight. let now: u64; // SAFETY: Reading the architectural counter has no side effects. @@ -318,10 +283,10 @@ pub(crate) fn build( } put64(&mut hw_img, hwdata::TIMESTAMPS, va::TIMESTAMPS)?; put32(&mut glob, globals::UNK_980, 100)?; - // The device tree's standby time (a machine with a firmware file; the templates' machine + // The device tree's standby time (a device-tree SoC; the templates' machine // has none). let dt_standby = match cfg.board { - hw::Board::Blob { .. } => { + hw::Board::DeviceTree(_) => { let prop = board::words::PROPS[board::words::STANDBY_TIMER]; match dev .of_node() @@ -369,8 +334,8 @@ pub(crate) fn build( globals::IDLE_PREDICTIVE, hw::tuning::IDLE_PREDICTIVE as u32, )?; - // HardwareDataA at +0x4580: maintained S8 request (the templates' machine; a board image - // from a file carries its own machine's words). + // HardwareDataA at +0x4580: maintained S8 request (the templates' machine; a record built + // from the device tree carries its own state words). if cfg.board == hw::Board::Templates { for (offset, value) in [ (0x2c, 8), diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index e659a507477b5..af53ff5a9d356 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -1,14 +1,17 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Test the actual J516C record builder's narrow zero-word exemption. +"""Test the actual J516C record builder's narrow zero-word exemption and its word sources. Uses synthetic public data and temporary host copies of production Rust modules. No GPU, firmware file, private fixture, kernel build or hardware access. The golden T6030 inputs below are from SiliconForest 9f09c85f8d3b, except the full hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes and again when the initial PB posting became a module parameter (the tuning -constant PB_BLOCKS removed). Its T6030 configuration module is independently -pinned to the original bytes. +constant PB_BLOCKS removed), and when the T6031 board source became the +device tree with reference words instead of a board file (the Board enum; +the T6030 tables are unchanged). Its T6030 configuration module is +independently pinned to the original bytes. prepare_board() is pinned too; +it was refreshed for the same Board change (its T6030 arm is unchanged). """ import argparse @@ -24,13 +27,13 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "658c58026553c098b899aeffa0b6ae5be2b14921aef19df9e4f14789b7b890c4", + "hw.rs": "2128be7e14bddfdc8d57951149c1fa4a276971e17bf8b6960fda0674c1fabfbe", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { ("hw.rs", "pub(crate) mod t6030 {"): "0079ae08fbe0e51cbf14e48711981a81048d1a1d3fc5495ab8d5c376283a760c", ("initdata.rs", "fn templates("): "c3115c9926d3d7d93317cb8510e93b04f30b0c256811d7137568e822dd93b8ee", - ("initdata.rs", "pub(crate) fn prepare_board("): "c59f968f125397efc63f75a49d3a8543c15c1fd4d50362f15683563398e64bc6", + ("initdata.rs", "pub(crate) fn prepare_board("): "ebef64ab87ef7b1f994eb7fa2a2424d05bc4fc9e88cbfbf18e51570bdfc4222d", ("board.rs", "pub(crate) fn from_adt("): "cecd0b28db5bd94d3b75c1bbb5f35204e8bf0699f74e5b3d08228c59025f6ac2", } @@ -51,7 +54,7 @@ def function(source, marker): use super::*; // Frozen pre-5131caff FROM_FILE offsets from SiliconForest 9f09c85f8d3b. - // This list is deliberately independent of both new lists. + // This list is deliberately independent of the three new lists. const OLD: [usize; 74] = [ 224,232,244,248,252,256,260,264,268,728,732,736,740,744,748, 992,1000,1012,1016,1024,1036,1496,1500,1504,1508,1512,1516, @@ -62,17 +65,32 @@ def function(source, marker): 45908,45912,45932,45940,45948,45956, ]; + const REF: Reference = Reference { + max_power_mw: 54321, + relative_power: [0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007], + fast_die_mask: 0x2111_2222_2333_2444, + alarm_mask: 0x2555_2666_2777_2888, + }; + + fn rows() -> [[u8; 12]; SENSOR_ROW_COUNT] { + [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"].map(|k| { + let mut r = [0x24u8; 12]; + r[..4].copy_from_slice(k); + r + }) + } + + fn row_words() -> Vec { + (SENSOR_ROWS..SENSOR_ROWS + 12 * SENSOR_ROW_COUNT).step_by(4).collect() + } + fn synthetic() -> (Vec, Vec, [Opp; 1], [u32; PROPS.len()]) { let mut file = vec![0u8; SIZE]; for (n, &at) in OLD.iter().enumerate() { put32(&mut file, at, 0x13570000 | (n as u32 + 1)).unwrap(); } - put32(&mut file, MAX_POWER, 54321).unwrap(); - for (n, key) in [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"].iter().enumerate() { - put32(&mut file, SENSOR_ROWS + 12*n, u32::from_le_bytes(**key)).unwrap(); - } // Public, synthetic OPPs: state0 off; eight voltage groups, five paired. - let mut main = vec![Opp {mhz:0, mv:[125;4], cells:4, mw:0}]; + let mut main = vec![Opp {mhz:0, mv:[125;4], cells:4}]; for (mv, freqs) in [ (600, &[300u32][..]), (650, &[500][..]), (700, &[700][..]), (750, &[800,900][..]), (800, &[950,1000][..]), @@ -80,10 +98,10 @@ def function(source, marker): (950, &[1250,1300][..]), ] { for &mhz in freqs { - main.push(Opp {mhz, mv:[mv;4], cells:4, mw:mhz*40}); + main.push(Opp {mhz, mv:[mv;4], cells:4}); } } - let cs = [Opp {mhz:24, mv:[635,0,0,0], cells:1, mw:0}]; + let cs = [Opp {mhz:24, mv:[635,0,0,0], cells:1}]; let mut props = [1u32; PROPS.len()]; props[PERIOD_MS] = 8; props[AVG_TC_MS] = 32; @@ -100,14 +118,16 @@ def function(source, marker): fn inputs<'a>(main: &'a [Opp], cs: &'a [Opp], props: [u32; PROPS.len()]) -> Inputs<'a> { Inputs {props, zones:&[], main, cs, afr:cs, slots:&[], empty_writable:&[], ttbat:0x100000, usc_base:0x1000000000, user_end:0x2000000000, - timestamps:0xfffffc2ffc000000, max_power_mw:54321} + timestamps:0xfffffc2ffc000000, sensor_rows: rows(), reference: &REF} } #[test] fn old74_is_exact_disjoint_partition() { - assert_eq!(FROM_FILE.len(), 32); + assert_eq!(REFERENCE_WORDS.len(), 14); + assert_eq!(row_words().len(), 18); assert_eq!(UNNEEDED.len(), 42); - let mut union = FROM_FILE.to_vec(); + let mut union = REFERENCE_WORDS.to_vec(); + union.extend(row_words()); union.extend_from_slice(&UNNEEDED); union.sort_unstable(); assert!(union.windows(2).all(|w| w[0] < w[1]), "duplicate or overlapping offsets"); @@ -117,30 +137,30 @@ def function(source, marker): } #[test] - fn actual_from_file_copies_only_retained32() { - let (file, _, _, _) = synthetic(); - let mut out = vec![0u8; SIZE]; - from_file(&mut out, &file).unwrap(); - for at in (0..SIZE).step_by(4) { - let expected = if FROM_FILE.contains(&at) {get32(&file, at).unwrap()} else {0}; - assert_eq!(get32(&out, at).unwrap(), expected, "copy offset +{at:#x}"); - } - } - - #[test] - fn actual_builder_leaves42_zero_and_preserves32() { + fn actual_builder_leaves42_zero_and_writes_the_32_from_their_sources() { let (file, main, cs, props) = synthetic(); - let mut out = vec![0u8; SIZE]; - from_file(&mut out, &file).unwrap(); + // Whatever the buffer held is not carried into the record. + let mut out = file.clone(); build(&mut out, &inputs(&main, &cs, props)).unwrap(); for &at in UNNEEDED.iter() { assert_ne!(get32(&file, at).unwrap(), 0, "fixture must exercise a real difference"); assert_eq!(get32(&out, at).unwrap(), 0, "built offset +{at:#x}"); } - for &at in FROM_FILE.iter() { - assert_eq!(get32(&out, at).unwrap(), get32(&file, at).unwrap(), "retained +{at:#x}"); + let mut reference = Vec::new(); + for r in REF.relative_power { reference.push(r); } + reference.push(REF.max_power_mw); + for m in [REF.fast_die_mask, REF.fast_die_mask, REF.alarm_mask] { + reference.push(m as u32); + reference.push((m >> 32) as u32); + } + for (&at, &want) in REFERENCE_WORDS.iter().zip(reference.iter()) { + assert_eq!(get32(&out, at).unwrap(), want, "reference +{at:#x}"); + } + let flat: Vec = rows().concat(); + for (k, &at) in row_words().iter().enumerate() { + assert_eq!(&out[at..at + 4], &flat[4 * k..4 * k + 4], "sensor row +{at:#x}"); } - // Model a matching board file with nonzero legacy values only at the42. + // Model a matching machine record with nonzero legacy values only at the42. let mut matching = out.clone(); for &at in UNNEEDED.iter() { put32(&mut matching, at, get32(&file, at).unwrap()).unwrap(); @@ -170,7 +190,7 @@ def function(source, marker): MUTANTS = { "broad-cross-check-exemption": ("cross_check", "UNNEEDED.contains(&at)", "true"), "nonzero-built-exempted": ("cross_check", "b == 0 && UNNEEDED.contains(&at)", "UNNEEDED.contains(&at)"), - "legacy-copy-reintroduced": ("from_file", "FROM_FILE.iter()", "FROM_FILE.iter().chain(UNNEEDED.iter())"), + "stale-buffer-kept": ("build", "out.fill(0);", ""), } @@ -248,7 +268,8 @@ def run(name, record): mutant = run(name, changed) if not mutant.returncode: raise RuntimeError("mutant survived: " + name) - regression = ("actual_from_file_copies_only_retained32" if func == "from_file" + regression = ("actual_builder_leaves42_zero_and_writes_the_32_from_their_sources" + if func == "build" else "actual_cross_check_exempts_only_zero_at_exact42") if regression + " ... FAILED" not in mutant.stdout: raise RuntimeError("new regression did not catch mutant: " + name) From a18d9316c9af1dae100dbdf0ca1e4a266c4ace0e Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Wed, 28 Oct 2026 05:02:38 +0530 Subject: [PATCH 15/46] tools/asahi: g15-maintained-handoff-tests: define the sensor-row count The handoff test compiles GpuAdt and the T6030 ADT parser from adt.rs on the host. GpuAdt now carries the T6031 sensor rows, sized by board::record::SENSOR_ROW_COUNT, which the generated source did not define, so the test no longer compiled. Take that production constant from board/record.rs into a generated record module, and hash board/record.rs with the other sources in the receipt. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit 23a7bd131f1b9978d23038bdd61f46479bc3b51d) Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/g15-maintained-handoff-tests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/asahi/g15-maintained-handoff-tests.py b/tools/asahi/g15-maintained-handoff-tests.py index 4dd2961d3a8ac..0c07c2eda3065 100644 --- a/tools/asahi/g15-maintained-handoff-tests.py +++ b/tools/asahi/g15-maintained-handoff-tests.py @@ -156,13 +156,16 @@ def main(): p = argparse.ArgumentParser(description=__doc__); p.add_argument('--output', type=Path); p.add_argument('--mutations', action='store_true'); a=p.parse_args() out=(a.output or Path(tempfile.mkdtemp(prefix='g15-maintained-handoff-'))).resolve(); out.mkdir(parents=True, exist_ok=True) adt=(G15/'adt.rs').read_text(); boot=(G15/'boot.rs').read_text(); hw=(G15/'hw.rs').read_text() + record=(G15/'board/record.rs').read_text() helpers='\n'.join(item(adt,m) for m in ["pub(crate) struct GpuAdt", "struct Adt<'a>", 'fn le32(', 'fn le64(', 'fn strncmp_eq(', "impl Adt<'_>", 'fn collect_perf(', 'fn parse(', 'fn admit_regions(']) constants='\n'.join(re.search(r'pub\(crate\) const '+n+r':[^;]+;', hw).group(0) for n in ['PAGE','PA_MASK','TTBS_SIZE','TABLES_SIZE','HANDOFF_SIZE','SHARED_L2_SIZE','UPPER']) + # GpuAdt carries the T6031 sensor rows, sized by the record builder's production constant. + record_constants=re.search(r'pub\(crate\) const SENSOR_ROW_COUNT:[^;]+;', record).group(0) start=hw.index(' firmware: &['); start=hw.index('[',start); end=start+1; depth=1 while depth: depth+=(hw[end]=='[')-(hw[end]==']'); end+=1 firmware='static BUILDS: &[FirmwareBuild] = &'+hw[start:end]+';\n'+item(hw,'pub(crate) struct FirmwareBuild') constructor=item(boot,' fn new(dev: &device::Device, shared: &Arc)') - source=PREFIX+'\nmod hw {\n'+constants+'\n}\n'+helpers+'\n'+firmware+'\nimpl Rtkit {\n'+constructor+'\n}\n'+CASES + source=PREFIX+'\nmod hw {\n'+constants+'\n}\nmod record {\n'+record_constants+'\n}\n'+helpers+'\n'+firmware+'\nimpl Rtkit {\n'+constructor+'\n}\n'+CASES variants={'actual':source} if a.mutations: variants['skip-owner-check']=source.replace('if !glue::reserved_uat(i as u32, r.start, r.end - r.start) {','if false {') @@ -170,7 +173,7 @@ def main(): variants['drop-reserved-ram']=source.replace('Some(dram_base..dram_base + dram_size), true)?;', 'Some(dram_base..dram_base + dram_size), false)?;') variants['require-obsolete-b0-profile']=source.replace(' FirmwareBuild {\n name: "J516S B0', ' #[cfg(CONFIG_SILICONFOREST_NATIVE_GPU)]\n FirmwareBuild {\n name: "J516S B0') variants['named-mailbox']=source.replace('cookie, core::ptr::null(), 0, &RTKIT_OPS', 'cookie, c"mbox".as_ptr(), 0, &RTKIT_OPS') - receipt={'scope':__doc__,'sources':{p:hashlib.sha256(t.encode()).hexdigest() for p,t in [('adt.rs',adt),('boot.rs',boot),('hw.rs',hw)]},'variants':{}} + receipt={'scope':__doc__,'sources':{p:hashlib.sha256(t.encode()).hexdigest() for p,t in [('adt.rs',adt),('boot.rs',boot),('hw.rs',hw),('board/record.rs',record)]},'variants':{}} for name,text in variants.items(): if name!='actual' and text==source: raise SystemExit('NOT_APPLIED '+name) src=out/(name+'.rs');src.write_text(text); binary=out/name From bcb02253d48695cdc7ec1ee4f4a6a350d468b8dc Mon Sep 17 00:00:00 2001 From: Naeem Malik Date: Wed, 28 Oct 2026 06:55:55 +0530 Subject: [PATCH 16/46] drm/asahi: g15: t6031: log the J516C hardware record digest Log the SHA-256 of the hardware record built from the device tree, the ADT sensor rows and the reference words, as the T6030 templates path does for a record that differs from its reference. A report from another J516C then shows at once whether its record matches. Also bring the prepare_board() comment up to date: the T6031 arm no longer has a digest to check or a firmware file to request. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Naeem Malik (cherry picked from commit 79f789160c73e1413cf817a1fb4c0beb8ac914f7) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/initdata.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/initdata.rs b/drivers/gpu/drm/asahi/g15/initdata.rs index c085d455f7a40..e53cb44eff43a 100644 --- a/drivers/gpu/drm/asahi/g15/initdata.rs +++ b/drivers/gpu/drm/asahi/g15/initdata.rs @@ -180,6 +180,7 @@ fn device_tree_board( max_power_mw, max_mhz ); + dev_info!(dev, "board data sha256 {}\n", Hex(&glue::sha256(&hw_img))); let max_khz = max_mhz.checked_mul(1000).ok_or_else(|| { dev_err!(dev, "board data: top operating point {} MHz does not fit in kHz\n", max_mhz); EINVAL @@ -190,8 +191,9 @@ fn device_tree_board( /// Verified board bytes, prepared before any GPU firmware is started. pub(crate) type BoardImages = (KVVec, KVVec, KVVec, u32); -/// Check availability, digest, layout and machine-derived tables before firmware start. -/// Keep the verified bytes so a later firmware request cannot change the admitted data. +/// Build and check the board images before any GPU firmware is started: the templates against +/// their digests and layout, or the device-tree record from the GPU node, the ADT and the SoC's +/// reference words. pub(crate) fn prepare_board( dev: &device::Device, cfg: &Config, From c45c861d1b28664326875dbac3cb27f19c1dc047 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Sun, 25 Oct 2026 23:12:44 +0100 Subject: [PATCH 17/46] drm/asahi: wrap G15 stamps through zero Include zero in the firmware stamp cycle. Keep absent dependencies and completed progress separate from wrapped zero stamps. Generated-by: LLM Signed-off-by: Eryk Wieliczko (cherry picked from commit d82144ad56a8ac7e75d74294e051683efccbb2a0) Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> (cherry picked from commit 7ccdfe85843df16b40bbb5f0080b2d288ae0b8bc) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/encode.rs | 5 +-- drivers/gpu/drm/asahi/g15/engine.rs | 16 +++----- drivers/gpu/drm/asahi/g15/poll.rs | 33 ++++++++------- drivers/gpu/drm/asahi/g15/recovery/logic.rs | 41 +++++++++++++++---- .../selftests/drivers/gpu/asahi-g15/run.py | 17 ++++++++ 5 files changed, 75 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/encode.rs b/drivers/gpu/drm/asahi/g15/encode.rs index e46ecd05bdf78..64aebf3254891 100644 --- a/drivers/gpu/drm/asahi/g15/encode.rs +++ b/drivers/gpu/drm/asahi/g15/encode.rs @@ -64,10 +64,9 @@ const fn align(n: u64, a: u64) -> u64 { } /// Firmware stamp value of ordinal `n` on the queue with this seed. -/// Stamps step by 0x100 through 0xffffff values and never read 0. +/// Stamps step by 0x100, including zero when the counter wraps. pub(crate) const fn stamp_seed(seed: u32, n: u64) -> u32 { - let base = (seed / 0x100) as u64; - ((((base - 1 + n % 0xffffff) % 0xffffff) + 1) * 0x100) as u32 + seed.wrapping_add((n as u32).wrapping_mul(0x100)) } /// Compute stamp of ordinal `n`. diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 6dd018369b478..da0c86f00e738 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -155,10 +155,10 @@ pub(crate) struct Flight { pub(crate) fs_start: u32, /// Channel producers (TA, FS, compute) after the prefix's records. pub(crate) rec: [u32; 3], - /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none): + /// Compute member j's FW stamp target, and the FS target it waits on directly: /// its predecessors in the prefix (for the compute-hang check). pub(crate) cstamp: [u32; sizes::COMPUTE_SLOTS], - pub(crate) cwait: [u32; sizes::COMPUTE_SLOTS], + pub(crate) cwait: [Option; sizes::COMPUTE_SLOTS], /// Compute slots, pass slots and clustering buffers it holds. pub(crate) claims: Claims, /// The prefix rebuilds the render allocators (priming); nothing is published behind it. @@ -189,7 +189,7 @@ impl Flight { fs_start: 0, rec: [0; 3], cstamp: [0; sizes::COMPUTE_SLOTS], - cwait: [0; sizes::COMPUTE_SLOTS], + cwait: [None; sizes::COMPUTE_SLOTS], claims: Claims::new(), rebuild: false, doomed: false, @@ -387,7 +387,7 @@ pub(crate) struct Engine { pub(crate) chan_wait: bool, /// Progress tracking of the oldest prefix. pub(crate) progress_flight: u64, - pub(crate) progress: [u32; 3], + pub(crate) progress: [Option; 3], pub(crate) stats: Stats, /// Phase timers (debug_flags bit 49). pub(crate) prof: Prof, @@ -425,7 +425,7 @@ impl Engine { rebuild_live: false, chan_wait: false, progress_flight: 0, - progress: [0; 3], + progress: [None; 3], stats: Stats::default(), prof: Prof::default(), debugfs: None, @@ -1191,11 +1191,7 @@ impl Firmware { } let j = pf.computes as usize; f.cstamp[j] = t; - f.cwait[j] = if pf.prev == Prev::Render { - pf.last_fs - } else { - 0 - }; + f.cwait[j] = (pf.prev == Prev::Render).then_some(pf.last_fs); g.ring.w64(8 * pf.c_head as usize, s.work.va)?; pf.c_head = (pf.c_head + 1) % sizes::QUEUE_CAP; lap.lap(&mut self.engine.prof.write_ns); diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 4e27f8ff60e99..71da7d86dc6e1 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -189,10 +189,7 @@ impl Shared { } } -/// Stamp cell `obs` reached its own target, bounded by the newest published target `tail`. -pub(crate) fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { - obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) -} +pub(crate) use super::recovery::logic::stamp_reached; /// Cursor `c` of a ring of `m` entries passed end `h` of entries from `b`, within tail `t`. pub(crate) fn ring_passed(c: u32, b: u32, h: u32, t: u32, m: u32) -> bool { @@ -498,23 +495,25 @@ impl Firmware { return; }; let (g, r, e) = (&self.graph.compute, &self.graph.render, &self.engine); - let (mut c, mut ta, mut fs) = (0, 0, 0); + let pending = super::recovery::logic::pending_stamp; + let (mut c, mut ta, mut fs) = (None, None, None); if f.computes != 0 { - c = g.fw_stamp.r32(0).unwrap_or(0); - if stamp_reached(c, f.target, e.target) { - c = 0; // its computes are done: later work does not count - } + c = g + .fw_stamp + .r32(0) + .ok() + .and_then(|v| pending(v, f.target, e.target)); } if f.renders != 0 { - ta = r.fw_stamp[0].r32(0).unwrap_or(0); - fs = r.fw_stamp[1].r32(0).unwrap_or(0); - if stamp_reached(ta, f.ta_target, e.ta_target) { - ta = 0; - } - if stamp_reached(fs, f.fs_target, e.fs_target) { - fs = 0; - } + ta = r.fw_stamp[0] + .r32(0) + .ok() + .and_then(|v| pending(v, f.ta_target, e.ta_target)); + fs = r.fw_stamp[1] + .r32(0) + .ok() + .and_then(|v| pending(v, f.fs_target, e.fs_target)); } let moved = e.progress_flight == f.published_ns && [c, ta, fs] != e.progress; let published = f.published_ns; diff --git a/drivers/gpu/drm/asahi/g15/recovery/logic.rs b/drivers/gpu/drm/asahi/g15/recovery/logic.rs index 6d36d1d6550d5..767fb7c952937 100644 --- a/drivers/gpu/drm/asahi/g15/recovery/logic.rs +++ b/drivers/gpu/drm/asahi/g15/recovery/logic.rs @@ -37,9 +37,18 @@ pub(crate) const DRAIN_POLLS: u32 = 1000; pub(crate) const FURTHER_HALTS: u32 = 4; /// Stamp cell `obs` reached the own target `own`, bounded by the newest published target -/// `tail`. Stamps are seeded so that a written stamp is never 0 and has a zero low byte. +/// `tail`. Written stamps have a zero low byte, including at counter wrap. pub(crate) const fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { - obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) + obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) +} + +/// The position of unfinished work; zero is a valid position after stamp wrap. +pub(crate) const fn pending_stamp(obs: u32, own: u32, tail: u32) -> Option { + if stamp_reached(obs, own, tail) { + None + } else { + Some(obs) + } } /// Cursor `c` of a ring of `m` entries passed the end `h` of entries starting at `b`, within @@ -73,9 +82,9 @@ pub(crate) struct Receipts { pub(crate) fs_start: u32, /// Host channel producers (TA, FS, compute) after the prefix's records. pub(crate) rec: [u32; 3], - /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none). + /// Compute member j's FW stamp target, and the FS target it waits on directly. pub(crate) cstamp: [u32; 16], - pub(crate) cwait: [u32; 16], + pub(crate) cwait: [Option; 16], } /// The newest published receipts. @@ -136,8 +145,10 @@ pub(crate) fn compute_hang(f: &Receipts, t: &Tail, o: &Observed) -> bool { if j == computes { return false; } - if f.cwait[j] != 0 && !stamp_reached(o.fs_stamp, f.cwait[j], t.fs_target) { - return false; // still waiting on a render of the prefix + if let Some(target) = f.cwait[j] { + if !stamp_reached(o.fs_stamp, target, t.fs_target) { + return false; // still waiting on a render of the prefix + } } if o.consumer[COMPUTE] != f.rec[COMPUTE] { return false; // its record not consumed yet @@ -332,10 +343,20 @@ mod tests { assert!(!stamp_reached(stamp(4), stamp(1), stamp(3))); // Across the u32 wrap. let own = 0xffffff00u32; + assert!(stamp_reached(0, own, 0x00000200)); + assert!(stamp_reached(0, 0, 0)); + assert!(!stamp_reached(0xffffff00, 0, 0x100)); assert!(stamp_reached(0x00000100, own, 0x00000200)); assert!(!stamp_reached(0xfffffe00, own, 0x00000200)); } + #[test] + fn progress_at_zero_wrap() { + assert_eq!(pending_stamp(0, 0x100, 0x200), Some(0)); + assert_eq!(pending_stamp(0, 0, 0x100), None); + assert_eq!(pending_stamp(0xffffff00, 0, 0x100), Some(0xffffff00)); + } + #[test] fn rings() { // Entries 10..14 of a 0x500 ring, tail at 20. @@ -401,13 +422,19 @@ mod tests { assert!(!compute_hang(&f, &t, &o)); o.compute_read = 15; // The running compute waits on an FS of the prefix that has not completed. - f.cwait[1] = 0x3d000300; + f.cwait[1] = Some(0x3d000300); o.fs_stamp = 0x3d000200; let mut t2 = t; t2.fs_target = 0x3d000400; assert!(!compute_hang(&f, &t2, &o)); o.fs_stamp = 0x3d000300; assert!(compute_hang(&f, &t2, &o)); + f.cwait[1] = Some(0); + t2.fs_target = 0x100; + o.fs_stamp = 0xffffff00; + assert!(!compute_hang(&f, &t2, &o)); + o.fs_stamp = 0; + assert!(compute_hang(&f, &t2, &o)); // No compute in the prefix. f.computes = 0; assert!(!compute_hang(&f, &t, &o)); diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 81883c315d27b..6b5b399cc395d 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -301,6 +301,23 @@ def main(): # checks that canonical normal/G17 UAPI remains unchanged; there is no global # Rust bindgen header selector in this integration. harness = RUST_PREFIX + f'\n#[path = "{G15 / "gate.rs"}"] mod gate;\n' + encode = (G15 / "encode.rs").read_text() + start = encode.index("pub(crate) const fn stamp_seed(") + end = encode.index("\n}", start) + 2 + harness += encode[start:end] + ''' +#[test] +fn firmware_stamp_wrap() { + for seed in [0xc1000000u32, 0x7a000000, 0x3d000000] { + let wrap = ((1u64 << 32) - seed as u64) / 0x100; + assert_eq!(stamp_seed(seed, 0), seed); + assert_eq!(stamp_seed(seed, wrap - 1), 0xffffff00); + assert_eq!(stamp_seed(seed, wrap), 0); + assert_eq!(stamp_seed(seed, wrap + 1), 0x100); + assert_eq!(stamp_seed(seed, 1 << 24), seed); + assert_eq!(stamp_seed(seed, u64::MAX), seed.wrapping_sub(0x100)); + } +} +''' harness += f'#[path = "{G15 / "recovery/logic.rs"}"] mod recovery;\n' harness += f'#[path = "{G15 / "power/idle.rs"}"] mod idle;\n' harness += f'#[path = "{G15 / "power/pstate.rs"}"] mod pstate;\n' From 2db2e9ffb62f75398cd3588abb019858d1c4a122 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Sun, 25 Oct 2026 23:12:44 +0100 Subject: [PATCH 18/46] drm/asahi: bind G15 slots at publication Idle VMs must not reserve hardware ASIDs. Bind without waiting at publication, release at successful retirement, and retain failed published jobs. Logical VM creation is limited by memory rather than slot count. Generated-by: LLM Signed-off-by: Eryk Wieliczko (cherry picked from commit 8525dd8e380a708356ec1ffaef0bbdf266cb328b) Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> (cherry picked from commit bb3ce596fa1e53595c929cab8e7748e2952ec111) [ryan: keep our two M4 handoff checks in Uat::bind_inner and the by-value robust_isolation read of Linux 7.3] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/engine.rs | 42 ++++++++++--- drivers/gpu/drm/asahi/g15/hw.rs | 2 - drivers/gpu/drm/asahi/g15/job.rs | 47 +++++++++++--- drivers/gpu/drm/asahi/g15/mod.rs | 2 +- drivers/gpu/drm/asahi/g15/poll.rs | 16 +++-- drivers/gpu/drm/asahi/g15/pools.rs | 6 +- drivers/gpu/drm/asahi/g15/recovery.rs | 6 +- drivers/gpu/drm/asahi/g15/uat.rs | 5 +- drivers/gpu/drm/asahi/g15/vm.rs | 89 +++++---------------------- drivers/gpu/drm/asahi/mmu.rs | 15 ++++- drivers/gpu/drm/asahi/slotalloc.rs | 17 +++++ 11 files changed, 143 insertions(+), 104 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index da0c86f00e738..50882313b6a3f 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -385,6 +385,8 @@ pub(crate) struct Engine { pub(crate) rebuild_live: bool, /// A ready prefix waits only for a record's consumption: repoll sooner. pub(crate) chan_wait: bool, + /// A ready job waits for an ASID held by a completing failed job. + pub(crate) slot_wait: bool, /// Progress tracking of the oldest prefix. pub(crate) progress_flight: u64, pub(crate) progress: [Option; 3], @@ -424,6 +426,7 @@ impl Engine { render_registered: false, rebuild_live: false, chan_wait: false, + slot_wait: false, progress_flight: 0, progress: [None; 3], stats: Stats::default(), @@ -546,6 +549,7 @@ impl Firmware { let mut result = Ok(()); self.engine.chan_wait = false; + self.engine.slot_wait = false; // Nothing is published while a hang request is armed. while self.recovery.failed().is_none() && !self.engine.frozen @@ -592,7 +596,7 @@ impl Firmware { dev_err!( self.graph.dev.as_ref(), "G15: publication of ctx {} failed after a queue WRITE ({:?})\n", - job.vm.slot, + job.slot(), e ); complete(done, &job, Err(EIO)); @@ -675,6 +679,15 @@ impl Firmware { /// yet (no free slot, or a channel record still unconsumed); EIO once a queue WRITE /// advanced; any other error means nothing was published. fn publish_prefix(&mut self, job: &Arc, done: &mut Completions) -> Result { + if let Err(error) = job.bind() { + if error == ENOSPC { + // A failed job can keep its binding until scheduler references drop after + // this pass. Leave healthy work pending and retry without the device lock. + self.engine.slot_wait = true; + return Err(EAGAIN); + } + return Err(error); + } let mut f = Flight::new(job.clone()); let result = self.publish_prefix_into(job, &mut f); match result { @@ -682,7 +695,11 @@ impl Firmware { // Never published: its storage is free again, and horizon jobs taken into it // fail with it. self.slots.release(&mut f.claims); + if job.progress.next() == job.progress.sent() { + job.release_binding(); + } for j2 in f.group.iter() { + j2.release_binding(); complete(done, j2, Err(e)); } } @@ -899,11 +916,20 @@ impl Firmware { hz_stop = Some(1); break; } - for k in 0..n2 { - match command(&j2, k).ok_or(EINVAL)? { - Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?, - Command::Render(_) => return Err(EINVAL), + j2.bind()?; + let encoded = (|| { + for k in 0..n2 { + match command(&j2, k).ok_or(EINVAL)? { + Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?, + Command::Render(_) => return Err(EINVAL), + } } + Ok(()) + })(); + if let Err(error) = encoded { + // Not in the horizon yet and nothing of it reached the firmware. + j2.release_binding(); + return Err(error); } computes += n2; j2.progress.set_sent(n2); @@ -1057,7 +1083,7 @@ impl Firmware { Queue, self.graph.dev, "G15: prefix ctx={} computes {} (n={} head {}) renders {} (draw {} ta/fs head {}/{}) behind {}: {:?}\n", - job.vm.slot, + job.slot(), computes, pf.n, pf.c_head, @@ -1095,7 +1121,7 @@ impl Firmware { ) -> Result { let n = pf.n + 1; let t = encode::stamp(n); - let ctx = job.vm.slot; + let ctx = job.slot(); let ts = &cmd.timestamps; let mut lap = Lap::start(); // Nothing published yet on failure. @@ -1236,7 +1262,7 @@ impl Firmware { let k = f.claims.pass[pf.renders as usize] as usize; let n = pf.draw + 1; let usc = job.queue.usc_base; - let ctx = job.vm.slot; + let ctx = job.slot(); let tta = encode::stamp_seed(ids::TA_SEED, n); let tfs = encode::stamp_seed(ids::FS_SEED, n); // Early tiling: a later render member after a render of the same prefix whose render diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index 4f2e9f6d3de1f..833f638e14c6c 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -512,8 +512,6 @@ pub(crate) mod sizes { pub(crate) const AHEAD_RECORDS: u32 = 64; /// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit). pub(crate) const MAX_QUEUES: u32 = 58; - /// TTBAT user slots: user VMs hold one each for their lifetime (ASIDs 1..63). - pub(crate) const MAX_VMS: u32 = 63; } /// Firmware channels, doorbells, seeds and identifiers. diff --git a/drivers/gpu/drm/asahi/g15/job.rs b/drivers/gpu/drm/asahi/g15/job.rs index 5f6669f06d1b0..e6f4cefd0a4ac 100644 --- a/drivers/gpu/drm/asahi/g15/job.rs +++ b/drivers/gpu/drm/asahi/g15/job.rs @@ -22,29 +22,35 @@ //! exactly once with [`Job::complete`] (later calls are ignored), through [`Completions`] //! collected under the device lock and [`complete_all`] after it is dropped. A job whose storage //! the firmware may still reference stays in `Engine::retained` (failed jobs until reboot): its -//! [`mmu::VmJobGuard`] keeps the VM's mappings and its VM state keeps the VM's TTBAT slot until +//! [`mmu::VmJobGuard`] keeps the VM's mappings and its binding keeps the VM's TTBAT slot until //! the last `Arc` drops. -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use kernel::{ bindings, c_str, device, dma_fence::{Fence, RawDmaFence}, drm::sched, + new_mutex, prelude::*, - sync::Arc, + sync::{Arc, Mutex}, }; use super::{command::Command, fence::JobFence, hw, queue::QueueState, vm::VmState, Shared}; use crate::mmu; /// A job as the engine sees it. +#[pin_data] pub(crate) struct Job { /// Submission ID (debugging). pub(crate) id: u64, /// The submitting queue (priority, service turn, horizon history). pub(crate) queue: Arc, - /// The job's VM (TTBAT slot, status). + /// The job's VM and status. pub(crate) vm: Arc, + /// Hardware context, assigned immediately before the engine publishes the job. + context: AtomicU32, + #[pin] + binding: Mutex>, /// Validated commands, in submission order (at most 64). pub(crate) commands: KVec, /// The job's hardware fence. @@ -73,11 +79,13 @@ impl Job { guard: mmu::VmJobGuard, ) -> Result> { let progress = super::engine::Progress::new(commands.len()); - Ok(Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { id, queue, vm, + context: AtomicU32::new(0), + binding <- new_mutex!(None, "g15::Job::binding"), commands, fence, progress, @@ -85,9 +93,29 @@ impl Job { _guard: guard, accepted: AtomicBool::new(false), started: AtomicBool::new(false), - }, + }), GFP_KERNEL, - )?) + ) + } + + pub(crate) fn slot(&self) -> u32 { + self.context.load(Ordering::Relaxed) + } + + /// Called under the device lock immediately before publication. + pub(crate) fn bind(&self) -> Result { + let mut slot = self.binding.lock(); + if slot.is_none() { + let binding = self.vm.try_bind()?; + self.context.store(binding.slot(), Ordering::Relaxed); + *slot = Some(binding); + } + Ok(()) + } + + /// All commands retired successfully, or nothing of the job was published. + pub(crate) fn release_binding(&self) { + drop(self.binding.lock().take()); } /// Completes the job: latches the VM status on failure, then signals the fence (after every @@ -120,6 +148,9 @@ impl Job { self.vm.status.record(error); } } + if result.is_ok() || !engine { + self.release_binding(); + } self.fence.signal(result); } diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 2a46b1637d03f..f468fb0a010eb 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -379,7 +379,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. + /// Live VMs and the client aliases of every user VM. pub(crate) vms: Arc, } diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 71da7d86dc6e1..7d4e2b9dd51ba 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -117,7 +117,7 @@ impl Shared { if let Err(error) = firmware.publish(&mut done) { self.fail_stop(error); } - if !firmware.engine.flights.is_empty() { + if !firmware.engine.flights.is_empty() || firmware.engine.slot_wait { self.rearm(firmware.engine.chan_wait); } } @@ -215,7 +215,7 @@ impl Firmware { // Idle bookkeeping of the PB: when the idle began, and the reclaim once it is due. self.note_idle(); self.reclaim_check(); - if self.recovery_idle() { + if self.recovery_idle() && self.engine.pending.is_empty() { return Ok(false); } } @@ -240,7 +240,10 @@ impl Firmware { // at the start of a pass that finds the flights empty. self.note_idle(); published?; - Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen) + Ok(!self.engine.flights.is_empty() + || self.engine.slot_wait + || self.recovery.needs_repoll() + || self.engine.frozen) } /// Drains the rings and retires finished flights, with up to four trailing 5 us polls @@ -459,6 +462,10 @@ impl Firmware { p.count.store(p.sent(), Ordering::Relaxed); } } + if result.is_ok() && p.next() == p.sent() { + // All published prefixes retired; another VM may run before this job resumes. + job.release_binding(); + } if p.next() < p.sent() || p.sent() < p.count() { return; // more of it published or still to publish } @@ -476,6 +483,7 @@ impl Firmware { match result { Err(error) => complete(done, j2, Err(error)), Ok(()) => { + j2.release_binding(); s.jobs += 1; s.job_cmds += p2.count() as u64; s.job_max = s.job_max.max(p2.count() as u64); @@ -619,7 +627,7 @@ impl Firmware { self.graph.dev.as_ref(), "G15: {}: ctx={} prefixes={} n={} target={:#x} user={:#x} fw={:#x} done={} read={} write={} head={} ch={}/{} events={}/{} errors={} busy={} fault={:#x} {}ms\n", what, - f.map_or(0, |f| f.job.vm.slot), + f.map_or(0, |f| f.job.slot()), e.flights.len(), e.ordinal, e.target, diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index 3364b136b9287..8a3989f723abb 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -624,8 +624,8 @@ impl Firmware { if others != 0 { dev_err!( self.dev(), - "PB backing: ctx {} maps {} chunks with {} flights of another VM\n", - vm.slot, + "PB backing: VM {} maps {} chunks with {} flights of another VM\n", + vm.id, self.pools.chunks.len(), others ); @@ -913,7 +913,7 @@ impl Firmware { .flights .iter() .enumerate() - .any(|(i, f)| self.receipts(i).renders != 0 && f.job.vm.slot == ctx); + .any(|(i, f)| self.receipts(i).renders != 0 && f.job.slot() == ctx); // At the maximum, after the refusals, the request is left unanswered: the poll pass // returns, and the host deadline and its recovery path apply to a stalled TA. if self.pools.ledger.exhausted() { diff --git a/drivers/gpu/drm/asahi/g15/recovery.rs b/drivers/gpu/drm/asahi/g15/recovery.rs index b81734b1d5f2d..e03b0d6661f93 100644 --- a/drivers/gpu/drm/asahi/g15/recovery.rs +++ b/drivers/gpu/drm/asahi/g15/recovery.rs @@ -224,7 +224,7 @@ impl Firmware { for flight in self.engine.flights.iter_mut().skip(1) { flight.doomed = true; } - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); let kind = if offset == logic::FRAGMENT_KILL { "fragment" } else { @@ -281,7 +281,7 @@ impl Firmware { if decision.compute_hang { self.report("compute hang"); } - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); if decision.class0_hang { dev_warn!( self.dev(), @@ -323,7 +323,7 @@ impl Firmware { /// anything is published; if it cannot be, the GPU stops. Anything else fail-stops. fn fail_or_recover(&mut self, recover: bool, fault: bool, done: &mut Completions) -> Result { let flight = !self.engine.flights.is_empty(); - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); self.report(if flight { "prefix FAILED" } else { diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index d6971489ee80f..ed84d5b350f85 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,8 +25,9 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` -//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) +//! and return them after successful retirement; failed published jobs retain their slots. +//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index 260f5e6b2cbb1..14d98bc4d50b1 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -13,16 +13,11 @@ //! whole PB range, whose backing chunks are mapped later, before the VM's prefixes are //! published (`Firmware::map_pb`) and, for the VM of the flights, when a chunk is backed //! ([`VmState::map_heap`], [`VmState::map_chunk`]); -//! - bind it to an exclusive TTBAT slot ([`mmu::Uat::bind`]) for as long as its [`VmState`] -//! lives; at most [`super::hw::sizes::MAX_VMS`] such VMs (ENOSPC beyond, checked before -//! binding so the slot allocator never blocks). +//! - leave it unbound until a runnable job needs a TTBAT slot. Each published job holds +//! its binding through completion; failed published jobs retain theirs until reboot. //! -//! The [`VmState`] of a VM is shared by the [`Registry`] (while the client's VM handle exists), -//! the VM's queues and its jobs. It cannot be owned by the [`mmu::Vm`] itself: the slot binding -//! holds a `Vm` reference. The registry drops its reference once the shared lifetime reports the -//! client's handle closed ([`mmu::Vm::is_closed`]), at the next VM creation; the slot returns when -//! the last queue or job of the VM is gone (failed jobs are retained until reboot, and their -//! VM's slot with them). +//! The registry keeps a VM's state while its client handle is open. Queues and jobs retain +//! closed VMs until their last reference drops. //! //! User page tables are cleaned to the point of coherency by the shared `pgtable.rs` (enabled by //! the adopted UAT); user objects are prepared at creation (`gpu::Gpu::prepare_new_objects`). @@ -30,7 +25,7 @@ //! alike. use core::ops::Range; -use core::sync::atomic::{AtomicU32, Ordering}; +use core::sync::atomic::Ordering; use kernel::{ drm::gem::BaseObject, new_mutex, @@ -39,7 +34,7 @@ use kernel::{ }; use super::{hw, object, Gpu}; -use crate::{debug::*, g17::status::VmStatus, gem, mmu, module_parameters}; +use crate::{debug::*, g17::status::VmStatus, gem, mmu}; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; @@ -123,20 +118,22 @@ struct HeapMappings { pub(crate) struct VmState { /// The `mmu::Vm` identifier (`Gpu::ids.vm`). pub(crate) id: u64, - /// The VM's TTBAT slot (ASID), for the work images' context. - pub(crate) slot: u32, /// The sticky status (shared with `file.rs`). pub(crate) status: Arc, /// The VM: span checks, stream reads and job guards. pub(crate) vm: mmu::Vm, - /// The exclusive slot binding, released before the live count drops. - bind: Option, + /// The adopted UAT, used to bind published jobs without waiting for a slot. + uat: Arc, registry: Arc, /// The PB heap chunks mapped in this VM. heap: Arc, } impl VmState { + pub(crate) fn try_bind(&self) -> Result { + self.uat.try_bind(&self.vm) + } + /// Drops the registry's references to closed VMs (the queues call this when destroyed). pub(crate) fn prune_registry(&self) { self.registry.prune(); @@ -199,39 +196,11 @@ impl VmState { } } -impl Drop for VmState { - fn drop(&mut self) { - mod_pr_debug!("G15: VM {} releases context {}\n", self.id, self.slot); - drop(self.bind.take()); - self.registry.live.fetch_sub(1, Ordering::AcqRel); - } -} - -/// Live-VM accounting and the VMs whose client handle is open. +/// The VMs whose client handle is open. #[pin_data] pub(crate) struct Registry { - /// VMs whose client handle may still be open, oldest first. #[pin] vms: Mutex>>, - /// Live [`VmState`]s (each holds a TTBAT slot) plus reservations being created. - live: AtomicU32, -} - -/// A reserved place among the live VMs, released unless a [`VmState`] takes it over. -struct Reservation<'a>(Option<&'a Registry>); - -impl Reservation<'_> { - fn into_state(mut self) { - self.0 = None; - } -} - -impl Drop for Reservation<'_> { - fn drop(&mut self) { - if let Some(registry) = self.0 { - registry.live.fetch_sub(1, Ordering::AcqRel); - } - } } impl Registry { @@ -239,32 +208,14 @@ impl Registry { Arc::pin_init( pin_init!(Self { vms <- new_mutex!(KVec::new(), "g15::Registry::vms"), - live: AtomicU32::new(0), }), GFP_KERNEL, ) } - /// Reserves a place for a new VM after dropping the registry's references to closed VMs. - fn reserve(&self) -> Result> { - self.prune(); - // The shared slot allocator hands out a single slot with robust isolation. - let limit = if module_parameters::robust_isolation.value() != 0 { - 1 - } else { - hw::sizes::MAX_VMS - }; - self.live - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |live| { - (live < limit).then_some(live + 1) - }) - .map_err(|_| ENOSPC)?; - Ok(Reservation(Some(self))) - } - /// Drops the registry's references to VMs whose client handle is closed. Called on VM close /// ([`closed`]), on queue destruction and before a new VM is created: a closed VM's page - /// tables, bound objects and TTBAT slot go as soon as its last queue or job is gone. + /// tables and bound objects go as soon as its last queue or job is gone. fn prune(&self) { let mut closed = KVec::new(); { @@ -367,7 +318,7 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { ClientAlias::collect(&firmware.objects)?, ) }; - let reservation = registry.reserve()?; + registry.prune(); let id = gpu.ids.vm.next(); let vm = gpu.uat.new_vm(id, kernel_range)?.with_tracked_status()?; let status = vm.status().ok_or(EINVAL)?.clone(); @@ -380,25 +331,19 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { }), GFP_KERNEL, )?; - // The context names the root only now: everything above is already visible to the walker. - let bind = gpu.uat.bind(&vm)?; - let slot = bind.slot(); - // From here on the state (even one that fails to allocate) gives the place back. - reservation.into_state(); let state = Arc::new( VmState { id, - slot, status, vm: vm.clone(), - bind: Some(bind), + uat: gpu.uat.clone(), registry: registry.clone(), heap, }, GFP_KERNEL, )?; registry.vms.lock().push(state, GFP_KERNEL)?; - mod_pr_debug!("G15: VM {} context {}\n", id, slot); + mod_pr_debug!("G15: VM {} created\n", id); Ok(vm) } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 12b0a7457fff3..a17128e6e574d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -3225,6 +3225,15 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, true) + } + + /// Binds a VM without waiting for another slot owner to retire. + pub(crate) fn try_bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, false) + } + + fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { self.inner.check_m4()?; let mut binding = vm.binding.lock(); self.inner.check_m4()?; @@ -3240,7 +3249,11 @@ impl Uat { None }); - let slot = self.slots.get(binding.bind_token)?; + let slot = if wait { + self.slots.get(binding.bind_token)? + } else { + self.slots.try_get(binding.bind_token)? + }; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs index fde7470fe5779..a504e7491bfda 100644 --- a/drivers/gpu/drm/asahi/slotalloc.rs +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -216,6 +216,20 @@ impl SlotAllocator { &self, token: Option, cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + ) -> Result> { + self.get_inner_impl(token, cb, true) + } + + /// Gets a slot without waiting; returns ENOSPC when every slot is held. + pub(crate) fn try_get(&self, token: Option) -> Result> { + self.get_inner_impl(token, |_a, _b| Ok(()), false) + } + + fn get_inner_impl( + &self, + token: Option, + cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + wait: bool, ) -> Result> { let mut inner = self.0.inner.lock(); @@ -256,6 +270,9 @@ impl SlotAllocator { } if oldest_time == u64::MAX { + if !wait { + return Err(ENOSPC); + } if first && inner.slot_limit == usize::MAX { pr_warn!( "{}: out of slots, blocking\n", From 2a3ddc79d8f29f9f64c4866532f6222c0097db25 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Mon, 26 Oct 2026 03:21:02 +0100 Subject: [PATCH 19/46] drm/asahi: lock G15 mappings once per submit Each range check and stream read relocked the VM and all external BOs. Share one exec lock across validation, preserving permission checks and cache maintenance, and propagate acquisition errors. In profiled 10-second vkQuake runs, kernel CPU time falls from 3.30 to 1.01 seconds. The bounded validation walk holds VM and external BO reservations together, so other users of shared buffers can wait. Generated-by: LLM Signed-off-by: Eryk Wieliczko (cherry picked from commit 610cc54ad4987dd4bf8948d44e1161b1a260b0e4) Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> (cherry picked from commit 289bcca8fd867448c3377072b00f4ec7b7ed441f) [ryan: our tree has no CONFIG_DRM_ASAHI_UAT_ADOPTION: build VmReadGuard, read_lock() and read_user_locked() with G15 or G16, the configurations that build Vm::read_user(), so the G16 kernel keeps compiling] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/command.rs | 39 ++++++++++++++----- drivers/gpu/drm/asahi/g15/gate.rs | 12 +++--- drivers/gpu/drm/asahi/mmu.rs | 25 +++++++++++- .../selftests/drivers/gpu/asahi-g15/run.py | 29 +++++++------- 4 files changed, 75 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 6fc1e03fb637c..66e52e6981197 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -146,6 +146,11 @@ pub(crate) struct Context<'a> { pub(crate) usc_base: u64, } +struct CheckedContext<'a> { + vm: &'a mut mmu::VmReadGuard, + usc_base: u64, +} + /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are /// checked ([`Prepared::validate`]). pub(crate) struct Prepared<'a> { @@ -377,7 +382,13 @@ impl Timestamps<'_> { /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a /// page without the access EACCES. -pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result { +pub(crate) fn cover( + vm: &mut mmu::VmReadGuard, + start: u64, + end: u64, + read: bool, + write: bool, +) -> Result { let mask = mmu::UAT_PGMSK as u64; if end <= start { return Err(EINVAL); @@ -407,7 +418,11 @@ pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) } /// Attachment spans, 128-byte granular, actually mapped with `write` or read access. -fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result { +fn check_attachments( + vm: &mut mmu::VmReadGuard, + list: &[uapi::drm_asahi_attachment], + write: bool, +) -> Result { for attachment in list { let end = attachment .pointer @@ -467,8 +482,8 @@ fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) { } /// Per-command render admission. -fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { - let vm = context.vm; +fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { + let vm = &mut *context.vm; let allowed = PROCESS_EMPTY_TILES | NO_VERTEX_CLUSTERING | RSRC_SPEC_HI @@ -552,11 +567,11 @@ fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Resu /// Per-command compute admission. fn check_compute( - context: &Context<'_>, + context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_compute, attachments: &[uapi::drm_asahi_attachment], ) -> Result { - let vm = context.vm; + let vm = &mut *context.vm; let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end); if cmd.flags != 0 || cmd.sampler_count != 0 @@ -650,6 +665,12 @@ impl Prepared<'_> { counts, timestamps, } = self; + // On error, unlock before the last timestamp mapping can drop and unmap. + let mut access = context.vm.read_lock()?; + let mut context = CheckedContext { + vm: &mut access, + usc_base: context.usc_base, + }; let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?; let mut compute = KVec::new(); let mut index = 0; @@ -658,12 +679,12 @@ impl Prepared<'_> { match entry.kind { SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => { // Validated readable, never serialized on G15. - check_attachments(context.vm, &attachment_list(entry.payload)?, false)?; + check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?; } SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?, CMD_RENDER => { let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; - check_render(context, &cmd)?; + check_render(&mut context, &cmd)?; commands.push( Command::Render(KBox::new( RenderCommand { @@ -678,7 +699,7 @@ impl Prepared<'_> { } CMD_COMPUTE => { let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; - check_compute(context, &cmd, &compute)?; + check_compute(&mut context, &cmd, &compute)?; let slots = resolved.get(index).ok_or(EINVAL)?; commands.push( Command::Compute(KBox::new( diff --git a/drivers/gpu/drm/asahi/g15/gate.rs b/drivers/gpu/drm/asahi/g15/gate.rs index a6fb20bf156b5..590f5e6f0a5c1 100644 --- a/drivers/gpu/drm/asahi/g15/gate.rs +++ b/drivers/gpu/drm/asahi/g15/gate.rs @@ -71,7 +71,7 @@ struct Window { impl Window { /// The `count` words at `va`, reading whole spans of up to 256 bytes (not past the page /// unless the record crosses it). Unreadable memory: EFAULT. - fn words(&mut self, vm: &mmu::Vm, va: u64, count: usize) -> Result<[u32; 10]> { + fn words(&mut self, vm: &mut mmu::VmReadGuard, va: u64, count: usize) -> Result<[u32; 10]> { let n = count * 4; let cached = self.len != 0 && va >= self.va @@ -108,7 +108,7 @@ struct Checked { /// One page check of `[start, end)` under the walk's page budget: ENOSPC once the budget is /// spent. -fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result { +fn check(vm: &mut mmu::VmReadGuard, start: u64, end: u64, count: &mut u32) -> Result { if *count >= CDM_MAX_PAGE_CHECKS { return Err(ENOSPC); } @@ -118,7 +118,7 @@ fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result { /// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable. /// The last proven program page is not checked again. -fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Result { +fn program(vm: &mut mmu::VmReadGuard, usc: u64, address: u64, checked: &mut Checked) -> Result { if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 { return Err(EINVAL); } @@ -134,7 +134,7 @@ fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Resul /// An indirect launch's 12 argument bytes: readable. The last proven argument page is not /// checked again; arguments that cross a page are checked each time. -fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result { +fn arguments(vm: &mut mmu::VmReadGuard, address: u64, checked: &mut Checked) -> Result { let end = address.checked_add(12).ok_or(EINVAL)?; let page = address & !(mmu::UAT_PGMSK as u64); let within = (end - 1) & !(mmu::UAT_PGMSK as u64) == page; @@ -149,7 +149,7 @@ fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result { } /// VDM admission: a normal root stream opens with exactly the G15 barrier. -pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { +pub(crate) fn vdm(vm: &mut mmu::VmReadGuard, va: u64) -> Result { if !params::stream_gate() { return Ok(()); } @@ -165,7 +165,7 @@ pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { /// CDM admission: walks the stream from its base through no-return links until TERMINATE, /// admitting only the known G15S records: direct launch (40), indirect launch (36/24), barriers /// (4), link (8). -pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { +pub(crate) fn cdm(vm: &mut mmu::VmReadGuard, usc: u64, base: u64) -> Result { if !params::stream_gate() { return Ok(()); } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index a17128e6e574d..df123e8ec64e8 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -411,6 +411,20 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +pub(crate) struct VmReadGuard(VmExecGuard); + +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +impl VmReadGuard { + pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { + Vm::covers_range_locked(&mut self.0, address, size, read, write) + } + + pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { + Vm::read_user_locked(&mut self.0, va, buf) + } +} + /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, @@ -2193,7 +2207,16 @@ impl Vm { /// mappings cannot change during the copy (GPUVM exec lock). #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - let mut inner = lock_vm(&self.gpuvm, None, false)?; + self.read_lock()?.read_user(va, buf) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + pub(crate) fn read_lock(&self) -> Result { + Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index 6b5b399cc395d..fa47cf416b072 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -114,6 +114,7 @@ pub const UAT_PGSZ: usize = 0x4000; pub const UAT_PGMSK: usize = UAT_PGSZ - 1; pub struct Vm(pub Vec); + pub type VmReadGuard = Vm; impl Vm { pub fn read_user(&self, va: u64, out: &mut [u8]) -> Result { let start = usize::try_from(va).map_err(|_| EFAULT)?; @@ -133,7 +134,7 @@ /// cover() calls: the page checks the walk made. pub static COVERS: Cell = Cell::new(0); } - pub fn cover(vm: &mmu::Vm, start: u64, end: u64, _: bool, _: bool) -> Result { + pub fn cover(vm: &mut mmu::VmReadGuard, start: u64, end: u64, _: bool, _: bool) -> Result { COVERS.with(|c| c.set(c.get() + 1)); if start >= SYNTHETIC { return Ok(()); @@ -156,34 +157,34 @@ } #[test] fn terminator_and_known_barrier() { - assert_eq!(gate::cdm(&vm(&[0x40000000]), 0, 0), Ok(())); - assert_eq!(gate::cdm(&vm(&[0x60000160, 0x40000000]), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&[0x40000000]), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&[0x60000160, 0x40000000]), 0, 0), Ok(())); } #[test] fn cyclic_links_are_rejected() { - assert_eq!(gate::cdm(&vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); } /// Words of 4 bytes that fill the byte budget: the root block and one per link. const BUDGET_WORDS: usize = 65 * 0x10000 / 4; #[test] fn byte_budget_requires_a_terminator() { - assert_eq!(gate::cdm(&vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); } #[test] fn final_record_may_terminate() { let mut words = vec![0x60000160; BUDGET_WORDS + 1]; words[BUDGET_WORDS - 1] = 0x40000000; - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); words[BUDGET_WORDS - 1] = 0x60000160; words[BUDGET_WORDS] = 0x40000000; - assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Err(EINVAL)); } #[test] fn more_than_8192_records_are_admitted() { // 1024 draws of ten records each, as a tessellated multi-draw writes them. let mut words = vec![0x60000160; 10240]; words.push(0x40000000); - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); } #[test] fn link_budget_still_bounds_the_walk() { @@ -193,13 +194,13 @@ words.extend([0x20000000, (i + 1) * 8]); } words.push(0x40000000); - assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Err(EINVAL)); let mut words = Vec::new(); for i in 0..64u32 { words.extend([0x20000000, (i + 1) * 8]); } words.push(0x40000000); - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); } /// The USC window of these cases and its first program, both in the mock's mapped range. const USC: u64 = crate::command::SYNTHETIC; @@ -216,7 +217,7 @@ } fn covers(words: &[u32]) -> (Result, u32) { crate::command::COVERS.with(|c| c.set(0)); - let r = gate::cdm(&vm(words), USC, 0); + let r = gate::cdm(&mut vm(words), USC, 0); (r, crate::command::COVERS.with(|c| c.get())) } const PAGE: u64 = mmu::UAT_PGSZ as u64; @@ -247,9 +248,9 @@ } #[test] fn malformed_and_unreadable_streams_are_rejected() { - assert_eq!(gate::cdm(&vm(&[0xffffffff]), 0, 0), Err(EINVAL)); - assert_eq!(gate::cdm(&vm(&[0x20000000, 2]), 0, 0), Err(EINVAL)); - assert_eq!(gate::cdm(&mmu::Vm(vec![]), 0, 0), Err(EFAULT)); + assert_eq!(gate::cdm(&mut vm(&[0xffffffff]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&[0x20000000, 2]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut mmu::Vm(vec![]), 0, 0), Err(EFAULT)); } } ''' From 6e5b91ab8ae1c22731122e7fdd65bffce4487c4e Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Mon, 26 Oct 2026 03:48:19 +0100 Subject: [PATCH 20/46] soc: apple: report T6030 GPU power to PMP The RTKit AGX vote does not set the PTD power request. PMP receives GPU performance-state updates but ignores its bandwidth demand without that request, leaving memory at its idle floor. Report AGX power after firmware and backend initialization. An earlier report is acknowledged but does not enable bandwidth scaling. Reassert it so a failed early power-off cannot leave a stale report across startup. Use the report driver's lock and checked soc-device table, and clear the report before releasing the vote. Other SoCs keep their existing path. With Mesa unchanged, the packaged kernel takes vkQuake from 104.5 to 372.5 FPS at 1920x1080 and compute-copy bandwidth from 32.7 to 136.3 GB/s. Tested on an M3 Pro J514S. Generated-by: LLM Signed-off-by: Eryk Wieliczko (cherry picked from commit 8824dd6a7aaa2b302ca7b3a77c2022bacc0fbdc8) Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> (cherry picked from commit 9bb74767dcd4eb3ad62605b5d26b0b758f6b3ca0) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/boot.rs | 1 + drivers/gpu/drm/asahi/g15/glue.c | 6 ++ drivers/gpu/drm/asahi/g15/mod.rs | 16 ++- drivers/pmdomain/apple/pmp-report.c | 56 +++++++++++ drivers/soc/apple/pmp-export.c | 40 +++++++- include/linux/soc/apple/pmp-report.h | 13 +++ include/linux/soc/apple/pmp.h | 6 ++ .../drivers/gpu/asahi-g15/pmp-vote.c | 98 +++++++++++++++++++ .../selftests/drivers/gpu/asahi-g15/run.py | 13 +++ 9 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index 95f5238fa805a..388391ec311f7 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -73,6 +73,7 @@ pub(crate) mod glue { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; pub(crate) fn asahi_g15_pmp_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_get(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + pub(crate) fn asahi_g15_pmp_ready() -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_put(dev: *mut core::ffi::c_void); fn asahi_g15_reserved_uat(index: u32, start: u64, size: u64) -> bool; } diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index 38359b39af3a7..8d6f3658322df 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -38,6 +38,7 @@ void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size); bool asahi_g15_dma_coherent(struct device *dev); int asahi_g15_pmp_prepare(struct device *dev); int asahi_g15_pmp_get(struct device *dev); +int asahi_g15_pmp_ready(void); void asahi_g15_pmp_put(struct device *dev); bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); @@ -153,6 +154,11 @@ int asahi_g15_pmp_get(struct device *dev) } +int asahi_g15_pmp_ready(void) +{ + return apple_pmp_t6030_gpu_ready(); +} + /* * Give back the vote of asahi_g15_pmp_get() after a start that failed before * the firmware was touched. The full M3 stack's J514S bridge has no release: diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index f468fb0a010eb..fa5ed4fbaa333 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -543,7 +543,21 @@ impl Firmware { vms, }, ); - if let Err(e) = firmware.init_graphs() { + let result = firmware.init_graphs().and_then(|_| { + if cfg.platform != hw::Platform::AdtWithPmp { + return Ok(()); + } + // A report before firmware startup is ACKed without enabling bandwidth scaling. + // Reassert it after setup, including the backend's final device control. + // SAFETY: This session retains its acknowledged T6030 AGX power vote. + let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; + if ret == 0 { + Ok(()) + } else { + Err(Error::from_errno(ret)) + } + }); + if let Err(e) = result { let Firmware { session, init, diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index f09b38aa310ca..ad9c29ae12325 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -1569,6 +1569,62 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, 50000); } +/* Find and check the report before taking the GPU vote; optionally update it. */ +static int apple_pmp_report_t6030_gfx(struct device *supplier, const bool *enable) +{ + struct apple_pmp_report *rep, *found = NULL; + struct device_node *pmp; + const u8 *table; + int len = 0, ret = -ENODEV; + + if (!IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) || + !of_machine_is_compatible("apple,t6030") || + !of_device_is_compatible(supplier->of_node, "apple,t6030-pmp-v2")) + return -ENODEV; + + /* T6030's ordered soc-device ID 5 is the acknowledged AGX request. */ + table = of_get_property(supplier->of_node, "apple,tunable-soc-device", &len); + if (!apple_pmp_devices_valid(table, len, BIT_ULL(4), BIT_ULL(4), true) || + memcmp(table + 4 * PMP_SOC_DEVICE_SIZE + 116, "AGX\0\0\0\0\0", 8)) + return -EINVAL; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->offsets != &apple_pmp_offsets_t6030) + continue; + pmp = of_parse_phandle(rep->dev->of_node, "apple,pmp", 0); + if (pmp == supplier->of_node) { + of_node_put(pmp); + if (found) + goto out; + found = rep; + } else { + of_node_put(pmp); + } + } + if (found) { + if (readq(found->base + found->offsets->status) != PMP_REPORT_READY) + ret = -EAGAIN; + else + ret = enable ? apple_pmp_report_set_state(found, 4, *enable, false) : 0; + } +out: + rcu_read_unlock(); + return ret; +} +int apple_pmp_report_check_t6030_gfx(struct device *supplier) +{ + return apple_pmp_report_t6030_gfx(supplier, NULL); +} +EXPORT_SYMBOL_GPL(apple_pmp_report_check_t6030_gfx); + +/* The RTKit AGX vote alone does not enable PMP's GPU bandwidth controllers. */ +int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) +{ + return apple_pmp_report_t6030_gfx(supplier, &enable); +} +EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx); + static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) { struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); diff --git a/drivers/soc/apple/pmp-export.c b/drivers/soc/apple/pmp-export.c index 120c4f32ae2b3..41091755a77c6 100644 --- a/drivers/soc/apple/pmp-export.c +++ b/drivers/soc/apple/pmp-export.c @@ -58,6 +58,7 @@ EXPORT_SYMBOL_GPL(apple_pmp_link_device); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) { + bool report_gfx; int ret; if ((command != 0x0e && command != 0x0f) || device_id > 15 || enabled > 1) @@ -69,6 +70,43 @@ int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) ret = apple_pmp_report_wait_supplier_ready(pmp_device, 5000); if (ret) return ret; - return apple_pmp_send_power_command(pmp_data, command, device_id, enabled); + /* T6030 needs both the RTKit power vote and the PTD bandwidth report. */ + report_gfx = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && + command == 0x0f && device_id == 5 && + of_machine_is_compatible("apple,t6030") && + of_device_is_compatible(pmp_device->of_node, "apple,t6030-pmp-v2"); + if (report_gfx) { + ret = apple_pmp_report_check_t6030_gfx(pmp_device); + if (ret) + return ret; + } + if (report_gfx && !enabled) { + ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); + if (ret) { + /* Clearing was written before the ACK timed out. Restore it. */ + apple_pmp_report_set_t6030_gfx(pmp_device, true); + return ret; + } + } + ret = apple_pmp_send_power_command(pmp_data, command, device_id, enabled); + /* Power is unknown after a lost reply; assert the report conservatively. */ + if (report_gfx && !enabled && ret) + apple_pmp_report_set_t6030_gfx(pmp_device, true); + return ret; } EXPORT_SYMBOL_GPL(apple_pmp_set_device_power); + +/* Reassert after startup, including a report left set by an earlier failure. */ +int apple_pmp_t6030_gpu_ready(void) +{ + int ret; + + guard(mutex)(&pmp_lock); + if (!pmp_data) + return -ENODEV; + ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); + if (ret) + return ret; + return apple_pmp_report_set_t6030_gfx(pmp_device, true); +} +EXPORT_SYMBOL_GPL(apple_pmp_t6030_gpu_ready); diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 5e819681ba106..dd3373517bf90 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -18,6 +18,9 @@ int apple_pmp_report_ane_available(const struct device_node *np); int apple_pmp_report_ane_perf(const struct device_node *np, struct device *owner, unsigned int level, bool *held); int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms); +/* Caller holds the PMP owner's vote lock and keeps the supplier bound. */ +int apple_pmp_report_check_t6030_gfx(struct device *supplier); +int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); @@ -40,6 +43,16 @@ static inline int apple_pmp_report_wait_supplier_ready(struct device *supplier, return -ENODEV; } +static inline int apple_pmp_report_check_t6030_gfx(struct device *supplier) +{ + return -ENODEV; +} + +static inline int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) +{ + return -ENODEV; +} + static inline int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout) { diff --git a/include/linux/soc/apple/pmp.h b/include/linux/soc/apple/pmp.h index 2317ed46f42de..8b329bf5d4dd5 100644 --- a/include/linux/soc/apple/pmp.h +++ b/include/linux/soc/apple/pmp.h @@ -8,6 +8,8 @@ struct device; #if IS_REACHABLE(CONFIG_APPLE_PMP) int apple_pmp_link_device(struct device *consumer); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled); +/* After the acknowledged AGX vote and successful GPU firmware startup. */ +int apple_pmp_t6030_gpu_ready(void); #else static inline int apple_pmp_link_device(struct device *consumer) { @@ -17,6 +19,10 @@ static inline int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enab { return -ENODEV; } +static inline int apple_pmp_t6030_gpu_ready(void) +{ + return -ENODEV; +} #endif /* Registration is private to the PMP Rust driver and its C bridge. */ int apple_pmp_register(struct device *dev, const void *data); diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c new file mode 100644 index 0000000000000..806148f64d1bf --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the production PMP vote transaction with failed firmware replies. */ +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +#define CONFIG_APPLE_T6030_GPU_PMP_LINK 1 +#define IS_ENABLED(x) (x) +#define guard(kind) ignore_lock +#define dev_warn(...) ((void)0) +static int pmp_lock; +static void ignore_lock(void *p) { (void)p; } +struct device { void *of_node; }; +static struct device owner; +static struct device *pmp_device = &owner; +static const void *pmp_data = &owner; +static bool t6030, requested; +static int ready, check, power_result[2], report_result[3], powers, reports; +static char calls[16]; +static unsigned int used; +static void called(char c) { assert(used < sizeof(calls) - 1); calls[used++] = c; } +static bool of_machine_is_compatible(const char *s) { (void)s; return t6030; } +static bool of_device_is_compatible(void *p, const char *s) +{ + (void)p; (void)s; return t6030; +} +static int apple_pmp_report_wait_supplier_ready(struct device *d, unsigned int ms) +{ + assert(d == &owner && ms == 5000); called('W'); return ready; +} +static int apple_pmp_send_power_command(const void *p, u8 cmd, u16 id, u32 enabled) +{ + assert(p == &owner && cmd == 0x0f && id == 5); + called(enabled ? 'P' : 'p'); assert(powers < 2); return power_result[powers++]; +} +static int apple_pmp_report_check_t6030_gfx(struct device *d) +{ + assert(d == &owner); called('V'); return check; +} +static int apple_pmp_report_set_t6030_gfx(struct device *d, bool enabled) +{ + assert(d == &owner); called(enabled ? 'R' : 'r'); requested = enabled; + assert(reports < 3); return report_result[reports++]; +} +#include "pmp-vote-production.h" +static void reset(void) +{ + pmp_data = &owner; t6030 = true; requested = false; ready = check = 0; powers = reports = used = 0; + memset(calls, 0, sizeof(calls)); + memset(power_result, 0, sizeof(power_result)); + memset(report_result, 0, sizeof(report_result)); +} +static void result(int expected, const char *sequence, bool enabled) +{ + assert(apple_pmp_set_device_power(0x0f, 5, enabled) == expected); + assert(strcmp(calls, sequence) == 0); +} +int main(void) +{ + reset(); + assert(apple_pmp_set_device_power(0, 5, 1) == -EINVAL); + assert(apple_pmp_set_device_power(0x0f, 16, 1) == -EINVAL); + assert(apple_pmp_set_device_power(0x0f, 5, 2) == -EINVAL); + assert(!used); + pmp_data = NULL; result(-ENODEV, "", true); + reset(); ready = -EAGAIN; result(-EAGAIN, "W", true); + reset(); t6030 = false; result(0, "WP", true); + reset(); t6030 = false; result(0, "Wp", false); + reset(); check = -EINVAL; result(-EINVAL, "WV", true); + reset(); result(0, "WVP", true); + /* Power-on must not start bandwidth tracking before firmware READY. */ + assert(!reports); + assert(apple_pmp_t6030_gpu_ready() == 0); + assert(strcmp(calls, "WVPrR") == 0 && requested); + reset(); requested = true; + assert(apple_pmp_t6030_gpu_ready() == 0); + assert(strcmp(calls, "rR") == 0 && requested); + reset(); pmp_data = NULL; + assert(apple_pmp_t6030_gpu_ready() == -ENODEV && !used); + reset(); report_result[0] = -ETIMEDOUT; + assert(apple_pmp_t6030_gpu_ready() == -ETIMEDOUT); + assert(strcmp(calls, "r") == 0); + reset(); report_result[1] = -ETIMEDOUT; + assert(apple_pmp_t6030_gpu_ready() == -ETIMEDOUT); + assert(strcmp(calls, "rR") == 0); + reset(); result(0, "WVrp", false); + reset(); power_result[0] = -ETIMEDOUT; result(-ETIMEDOUT, "WVP", true); + reset(); report_result[0] = -ETIMEDOUT; result(-ETIMEDOUT, "WVrR", false); + reset(); report_result[0] = -ETIMEDOUT; report_result[1] = -EIO; + result(-ETIMEDOUT, "WVrR", false); + reset(); power_result[0] = -EIO; result(-EIO, "WVrpR", false); + puts("production PMP vote: scope, sequencing, reply errors and rollback: PASS"); +} diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index fa47cf416b072..2ae5be4b57e56 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -298,6 +298,19 @@ def main(): str(Path(__file__).with_name("pmp-bridge.c")), "-o", str(output)], check=True) subprocess.run([str(output)], check=True) + owner = (ROOT / "drivers/soc/apple/pmp-export.c").read_text() + start = owner.index("int apple_pmp_set_device_power(") + end = owner.index("\n}", start) + 2 + vote = owner[start:end] + start = owner.index("int apple_pmp_t6030_gpu_ready(") + end = owner.index("\n}", start) + 2 + (temp / "pmp-vote-production.h").write_text(vote + "\n" + owner[start:end]) + output = temp / "pmp-vote" + subprocess.run([os.environ.get("HOSTCC", "cc"), "-std=c11", "-Wall", "-Wextra", + "-Werror", "-I", str(temp), + str(Path(__file__).with_name("pmp-vote.c")), "-o", str(output)], check=True) + subprocess.run([str(output)], check=True) + # Native wire layout is a private per-backend facade. Its independent host test # checks that canonical normal/G17 UAPI remains unchanged; there is no global # Rust bindgen header selector in this integration. From 0652b1423392c2cebd92b535485bc6ccc3224f00 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:28:20 -0700 Subject: [PATCH 21/46] tests/asahi: follow G15 publication binding interfaces Carry the host models forward with the publication-time binding API: provide Job::slot and a separate VM identity for PB growth, and pin the hw.rs golden after proving the only change removes MAX_VMS and its comment. Exercise production bind, release, completion and prefix-admission bodies with explicit modeled slot allocation. Cover retry on slot exhaustion, binding reuse, successful release, failed-job reference lifetime and an append failure after publication, with five additional runtime mutants. Model the T6030 PMP readiness service in the retention harness. Cover readiness failure with and without orphan allocation, successful startup, and exclusion on other platforms, with two additional runtime mutants. Retain the existing failure and removed-fix controls. Clarify last-reference versus fail-stop retention in the ownership comments and format the carried compute match arm. These edits change no production admission, firmware identity, power policy or submission behavior. Generated-by: LLM Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> (cherry picked from commit 9d5fd8f2edca673ce7fa547f99cdc5e9ba914b05) [ryan: keep our ENOENT, unobserved_teardown and old-control digest additions in the host tests, and pin our hw.rs bytes] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/engine.rs | 4 +- drivers/gpu/drm/asahi/g15/job.rs | 2 +- drivers/gpu/drm/asahi/g15/uat.rs | 3 +- drivers/gpu/drm/asahi/g15/vm.rs | 3 +- tools/asahi/g15-input-error-tests.py | 123 ++++++++++++++++++++++++--- tools/asahi/g15-pb-growth-tests.py | 8 ++ tools/asahi/g15-retention-tests.py | 64 +++++++++++--- tools/asahi/g15-zero-words-tests.py | 5 +- 8 files changed, 184 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 50882313b6a3f..754f5eac4ff36 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -920,7 +920,9 @@ impl Firmware { let encoded = (|| { for k in 0..n2 { match command(&j2, k).ok_or(EINVAL)? { - Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?, + Command::Compute(c) => { + self.prefix_compute(&j2, c, &mut pf, f, false)? + } Command::Render(_) => return Err(EINVAL), } } diff --git a/drivers/gpu/drm/asahi/g15/job.rs b/drivers/gpu/drm/asahi/g15/job.rs index e6f4cefd0a4ac..a93b6039cc466 100644 --- a/drivers/gpu/drm/asahi/g15/job.rs +++ b/drivers/gpu/drm/asahi/g15/job.rs @@ -21,7 +21,7 @@ //! The engine owns an `Arc` from [`super::Firmware::enqueue`] on. It completes the job //! exactly once with [`Job::complete`] (later calls are ignored), through [`Completions`] //! collected under the device lock and [`complete_all`] after it is dropped. A job whose storage -//! the firmware may still reference stays in `Engine::retained` (failed jobs until reboot): its +//! the firmware may still reference after a fail-stop stays in `Engine::retained` until reboot: its //! [`mmu::VmJobGuard`] keeps the VM's mappings and its binding keeps the VM's TTBAT slot until //! the last `Arc` drops. diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index ed84d5b350f85..53e99f5a772b2 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -26,7 +26,8 @@ //! redundant but harmless.) //! //! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) -//! and return them after successful retirement; failed published jobs retain their slots. +//! and return them after successful retirement. Failed published jobs retain their slots until +//! their last reference drops (until reboot for jobs retained after a fail-stop). //! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index 14d98bc4d50b1..24a912d5968bb 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -14,7 +14,8 @@ //! published (`Firmware::map_pb`) and, for the VM of the flights, when a chunk is backed //! ([`VmState::map_heap`], [`VmState::map_chunk`]); //! - leave it unbound until a runnable job needs a TTBAT slot. Each published job holds -//! its binding through completion; failed published jobs retain theirs until reboot. +//! its binding through completion; failed published jobs retain theirs until their last +//! reference drops (until reboot for jobs retained after a fail-stop). //! //! The registry keeps a VM's state while its client handle is open. Queues and jobs retain //! closed VMs until their last reference drops. diff --git a/tools/asahi/g15-input-error-tests.py b/tools/asahi/g15-input-error-tests.py index 809411963fd17..2dba24aa49204 100644 --- a/tools/asahi/g15-input-error-tests.py +++ b/tools/asahi/g15-input-error-tests.py @@ -2,8 +2,10 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT """Actual G15 job dependency completion and G17 status/policy bodies on host services. -DMA fence lookup, engine submission, queue-fence ordering and kernel mutexes are -models. No kernel scheduling, GPU access or firmware execution is exercised. +DMA fence lookup, engine submission, queue-fence ordering, VM slot allocation and +kernel mutexes are models. Production job binding/completion and prefix admission +bodies run against them. No kernel scheduling, GPU access or firmware execution +is exercised. """ import argparse import hashlib @@ -33,6 +35,7 @@ def extract(text, marker): type Result = std::result::Result; const EIO:Error=Error(-5);const ENODEV:Error=Error(-19);const ETIMEDOUT:Error=Error(-110); const ECANCELED:Error=Error(-125);const EINVAL:Error=Error(-22);const ENOMEM:Error=Error(-12);const ENOENT:Error=Error(-2); +const ENOSPC:Error=Error(-28);const EAGAIN:Error=Error(-11); macro_rules! pr_warn { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } macro_rules! mod_pr_debug { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } struct Mutex(StdMutex);impl Mutex{fn lock(&self)->MutexGuard<'_,T>{self.0.lock().unwrap()}} @@ -45,8 +48,17 @@ def extract(text, marker): mod g17 {use super::*;__POLICY__} mod bindings {pub unsafe fn dma_fence_get_status(p:*const i32)->i32{unsafe{*p}}} struct Fence(i32);impl Fence{fn raw(&self)->*const i32{&self.0}} -struct Vm{status:VmStatus} -struct Progress{signaled:AtomicBool} +struct Binding{slot:u32,drops:Arc} +impl Binding{fn slot(&self)->u32{self.slot}} +impl Drop for Binding{fn drop(&mut self){self.drops.fetch_add(1,Ordering::SeqCst);}} +struct Vm{status:VmStatus,refuse_binding:AtomicBool,next_slot:AtomicU32,allocations:AtomicU32,binding_drops:Arc} +impl Vm{fn try_bind(&self)->Result{ + if self.refuse_binding.load(Ordering::SeqCst){return Err(ENOSPC);} + self.allocations.fetch_add(1,Ordering::SeqCst); + Ok(Binding{slot:self.next_slot.load(Ordering::SeqCst),drops:self.binding_drops.clone()}) +}} +struct Progress{signaled:AtomicBool,next:AtomicU32,sent:AtomicU32} +impl Progress{fn next(&self)->u32{self.next.load(Ordering::SeqCst)}fn sent(&self)->u32{self.sent.load(Ordering::SeqCst)}} struct Guard{quarantines:AtomicU32}impl Guard{fn quarantine(&self){self.quarantines.fetch_add(1,Ordering::SeqCst);}} struct JobFence{vm:Arc,signals:AtomicU32,result:AtomicI32,saw_sticky:AtomicI32,saw_report:AtomicI32,saw_mirror:AtomicI32} impl JobFence{ @@ -61,12 +73,25 @@ def extract(text, marker): } struct Shared{crashed:AtomicBool,submits:AtomicU32,submit_result:Result} impl Shared{fn submit(&self,_:Arc)->Result{self.submits.fetch_add(1,Ordering::SeqCst);self.submit_result}} -struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence} +struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence,context:AtomicU32,binding:Mutex>} impl Job {__JOB__} +type Completions=Vec; +fn complete(done:&mut Completions,_job:&Arc,result:Result){done.push(result);} +struct Flight{claims:(),group:Vec>} +impl Flight{fn new(_job:Arc)->Self{Self{claims:(),group:vec![]}}} +struct Slots{releases:u32} +impl Slots{fn release(&mut self,_claims:&mut ()){self.releases+=1;}} +struct Engine{slot_wait:bool} +struct HostFirmware{engine:Engine,slots:Slots,publish_result:Result,publishes:u32} +impl HostFirmware{ + fn new(result:Result)->Self{Self{engine:Engine{slot_wait:false},slots:Slots{releases:0},publish_result:result,publishes:0}} + fn publish_prefix_into(&mut self,_job:&Arc,_flight:&mut Flight)->Result{self.publishes+=1;self.publish_result} + __PREFIX__ +} fn make_job(own:Option)->(Arc,Box){ - let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast())}); + let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast()),refuse_binding:AtomicBool::new(false),next_slot:AtomicU32::new(7),allocations:AtomicU32::new(0),binding_drops:Arc::new(AtomicU32::new(0))}); if let Some(error)=own{vm.status.record(error);} - let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false),next:AtomicU32::new(0),sent:AtomicU32::new(0)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)},context:AtomicU32::new(0),binding:Mutex(StdMutex::new(None))}); (job,mirror) } fn shared(crashed:bool,result:Result)->Arc{Arc::new(Shared{crashed:AtomicBool::new(crashed),submits:AtomicU32::new(0),submit_result:result})} @@ -90,7 +115,7 @@ def extract(text, marker): assert_eq!(job.fence.signals.load(Ordering::SeqCst),1,"dependency completion must signal once"); assert_eq!(job.vm.status.get(),0,"duplicate completion must not poison VM"); // A later job on the same VM is admitted despite the reported client error. - let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false),next:AtomicU32::new(0),sent:AtomicU32::new(0)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)},context:AtomicU32::new(0),binding:Mutex(StdMutex::new(None))}); later.start(&device,&[Fence(1)]);assert_eq!(device.submits.load(Ordering::SeqCst),1,"successful dependencies after report must submit"); } for own in [EIO,EINVAL]{for input in [ECANCELED,EIO,ENODEV,ETIMEDOUT,EINVAL]{ @@ -111,8 +136,70 @@ def extract(text, marker): } // An engine-owned fault still quarantines before completion. let(job,_mirror)=make_job(None);job.complete(Err(EIO));assert_eq!(job._guard.quarantines.load(Ordering::SeqCst),1);assert_eq!(job.vm.status.get(),EIO.to_errno()); + // Actual bind/release bodies: a job keeps its slot until explicitly released. + { + let(job,_mirror)=make_job(None); + job.vm.refuse_binding.store(true,Ordering::SeqCst); + assert_eq!(job.bind(),Err(ENOSPC));assert!(job.binding.lock().is_none()); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),0); + job.vm.refuse_binding.store(false,Ordering::SeqCst); + job.bind().unwrap();assert_eq!(job.slot(),7,"binding must publish its slot"); + job.vm.next_slot.store(9,Ordering::SeqCst); + job.bind().unwrap();assert_eq!(job.slot(),7,"repeated bind must keep existing binding"); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),1); + job.release_binding();job.release_binding(); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),1,"release_binding must drop exactly once"); + job.bind().unwrap();assert_eq!(job.slot(),9,"released job may acquire a different slot"); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),2); + } + // An engine failure must not recycle an ASID while retained job references live. + for result in [Ok(()),Err(EIO),Err(ETIMEDOUT),Err(ECANCELED)]{ + let(job,_mirror)=make_job(None);job.bind().unwrap();let drops=job.vm.binding_drops.clone(); + job.complete(result); + if result.is_err(){ + assert!(job.binding.lock().is_some(),"failed engine completion must retain binding"); + assert_eq!(drops.load(Ordering::SeqCst),0); + job.complete(Ok(()));assert!(job.binding.lock().is_some(),"duplicate completion must retain failed binding"); + let retained=job.clone();drop(job); + assert_eq!(drops.load(Ordering::SeqCst),0,"a retained reference must keep the binding"); + drop(retained);assert_eq!(drops.load(Ordering::SeqCst),1,"last reference releases failed binding"); + }else{ + assert!(job.binding.lock().is_none(),"successful engine completion releases binding"); + assert_eq!(drops.load(Ordering::SeqCst),1);drop(job);assert_eq!(drops.load(Ordering::SeqCst),1); + } + } + // Run the real prefix admission wrapper; only firmware publication is modeled. + { + let(job,_mirror)=make_job(None);let mut fw=HostFirmware::new(Ok(()));let mut done=vec![]; + job.vm.refuse_binding.store(true,Ordering::SeqCst); + assert_eq!(fw.publish_prefix(&job,&mut done),Err(EAGAIN)); + assert!(fw.engine.slot_wait,"slot exhaustion must request retry"); + assert_eq!(fw.publishes,0);assert!(done.is_empty());assert_eq!(job.vm.status.get(),0); + job.vm.refuse_binding.store(false,Ordering::SeqCst); + assert_eq!(fw.publish_prefix(&job,&mut done),Ok(()));assert_eq!(fw.publishes,1); + } + for published in [false,true]{ + let(job,_mirror)=make_job(None);job.bind().unwrap(); + job.progress.sent.store(u32::from(published),Ordering::SeqCst); + let mut fw=HostFirmware::new(Err(ENOMEM));let mut done=vec![]; + assert_eq!(fw.publish_prefix(&job,&mut done),Err(ENOMEM)); + assert_eq!(fw.slots.releases,1);assert!(done.is_empty()); + if published{ + assert!(job.binding.lock().is_some(),"published prefix must retain binding on append failure"); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),0); + }else{ + assert!(job.binding.lock().is_none(),"unpublished failure must release binding"); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),1); + } + } + { + let(job,_mirror)=make_job(None);let mut fw=HostFirmware::new(Err(EIO));let mut done=vec![]; + assert_eq!(fw.publish_prefix(&job,&mut done),Err(EIO)); + assert!(job.binding.lock().is_some(),"firmware-visible failure must retain binding"); + assert_eq!(fw.slots.releases,0); + } assert_eq!(unrelated.vm.status.get(),0,"unrelated VM untouched");assert_eq!(unrelated.vm.status.reported_error(),0);assert_eq!(other_mirror.load(Ordering::SeqCst),0);assert_eq!(unrelated.fence.signals.load(Ordering::SeqCst),0); - println!("PASS: actual Job::start/completion and VmStatus/policy: healthy/failed VMs, mirror ordering, one-shot completion, successful input, device faults and isolation"); + println!("PASS: production job/prefix/status bodies: input errors, binding reuse/release/retention, exhaustion retry, published-prefix lifetime and isolation"); } ''' @@ -130,17 +217,25 @@ def main(): current = (ROOT / job_path).read_text() status = (ROOT / "drivers/gpu/drm/asahi/g17/status.rs").read_text() policy = (ROOT / "drivers/gpu/drm/asahi/g17/dependency.rs").read_text() + engine = (ROOT / "drivers/gpu/drm/asahi/g15/engine.rs").read_text() methods = [extract(job, " pub(crate) fn complete("), extract(job, " fn finish("), extract(job, " fn start(self: &Arc")] # The old control needs the current helper for the duplicate-completion test; # it still runs its actual unmodified start, which reproduces the lost input error. methods.append(extract(job if "fn finish_dependency(" in job else current, " fn finish_dependency(")) + # Historical removed-fix controls predate publication bindings. Keep their + # actual start/completion bodies and use current binding helpers alongside. + binding_source = job if "fn release_binding(" in job else current + methods.extend(extract(binding_source, marker) for marker in [ + " pub(crate) fn slot(", " pub(crate) fn bind(", + " pub(crate) fn release_binding("]) bodies = "\n".join(methods) code = PRELUDE.replace("__JOB__", bodies).replace("__STATUS__", "\n".join( extract(status, marker) for marker in [" pub(crate) fn get(", " pub(crate) fn record(", " pub(crate) fn reported_error(", " pub(crate) fn report_failure(", " fn unobserved_teardown("])).replace( - "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")) + "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")).replace( + "__PREFIX__", extract(engine, " fn publish_prefix(")) variants = {"control": code} expected = {} if args.mutations: @@ -154,6 +249,11 @@ def main(): "success-refused": ("if let Some(status) = failed {", "if let Some(status) = failed.or(Some(-125)) {", "successful dependencies after report must submit"), "dependency-completes-twice": (" fn finish_dependency(&self, error: Error) {\n if self.progress.signaled.swap(true, Ordering::AcqRel) {\n return;\n }", " fn finish_dependency(&self, error: Error) {", "dependency completion must signal once"), "mirror-not-published": ("unsafe { AtomicI32::from_ptr(mirror.word) }.store(error, Ordering::SeqCst);", "let _ = mirror;", "mirror must precede fence signal"), + "failed-engine-releases-binding": ("if result.is_ok() || !engine {", "if true {", "failed engine completion must retain binding"), + "binding-release-leaks": ("drop(self.binding.lock().take());", "let _ = self;", "release_binding must drop exactly once"), + "bound-job-rebinds": ("if slot.is_none() {", "if true {", "repeated bind must keep existing binding"), + "slot-exhaustion-not-retried": ("self.engine.slot_wait = true;", "self.engine.slot_wait = false;", "slot exhaustion must request retry"), + "published-prefix-releases-binding": ("if job.progress.next() == job.progress.sent() {", "if true {", "published prefix must retain binding on append failure"), } for name, (old, new, message) in changes.items(): if code.count(old) != 1: @@ -162,7 +262,8 @@ def main(): expected[name] = message variants["control-after"] = code receipt = {"scope": __doc__, "ref": args.ref, "sources": {name: hashlib.sha256(text.encode()).hexdigest() - for name, text in [(job_path, job), ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} + for name, text in [(job_path, job), ("g15/engine.rs", engine), + ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} for name, source in variants.items(): path = args.output / (name + ".rs"); path.write_text(source) binary = args.output / name diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index 3bd600790b0e3..cbe1280141244 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -626,6 +626,7 @@ def consts(text, names): pub list: Mutex>, } pub struct VmState { + pub id: u64, pub slot: u32, pub vm: mmu::Vm, pub heap: Arc, @@ -633,6 +634,8 @@ def consts(text, names): impl VmState { pub fn new(slot: u32) -> Arc { Arc::new(VmState { + // Keep VM identity distinct from the modeled hardware slot. + id: 0x10000 + u64::from(slot), slot, vm: mmu::Vm::new(slot), heap: Arc::new(HeapMappings { list: Mutex::new(KVec(Vec::new())) }), @@ -642,6 +645,11 @@ def consts(text, names): pub struct Job { pub vm: Arc, } + impl Job { + pub fn slot(&self) -> u32 { + self.vm.slot + } + } pub struct Flight { pub job: Job, } diff --git a/tools/asahi/g15-retention-tests.py b/tools/asahi/g15-retention-tests.py index 129567ab373e0..f5c8c8dce3294 100644 --- a/tools/asahi/g15-retention-tests.py +++ b/tools/asahi/g15-retention-tests.py @@ -34,6 +34,8 @@ def item(text, marker): use std::{cell::{Cell, RefCell, RefMut}, mem::MaybeUninit, ops::{Deref, DerefMut}, sync::{Arc, atomic::{AtomicBool, Ordering}}}; type Error = i32; +trait FromErrno { fn from_errno(value: i32) -> Self; } +impl FromErrno for Error { fn from_errno(value: i32) -> Self { value } } type Result = std::result::Result; const GFP_KERNEL: u32 = 0; const EINVAL: Error = -22; @@ -43,6 +45,7 @@ def item(text, marker): static FAIL: Cell = const { Cell::new(0) }; static OWNER_DROPS: Cell = const { Cell::new(0) }; static BUFFER_DROPS: Cell = const { Cell::new(0) }; + static PMP_CALLS: Cell = const { Cell::new(0) }; } fn step(site: u32) -> Result { if FAIL.get() & (1 << site) != 0 { Err(-100 - site as i32) } else { Ok(()) } @@ -72,8 +75,13 @@ def item(text, marker): impl DerefMut for KBox { fn deref_mut(&mut self) -> &mut T { &mut self.0 } } struct AsahiDevice; impl AsahiDevice { fn as_ref(&self) -> &Self { self } } -struct Config; -static CONFIG: Config = Config; +mod hw { + #[derive(PartialEq)] + pub enum Platform { Adt, AdtWithPmp } +} +struct Config { platform: hw::Platform } +static CONFIG: Config = Config { platform: hw::Platform::Adt }; +static PMP_CONFIG: Config = Config { platform: hw::Platform::AdtWithPmp }; mod mmu { pub struct Uat; } mod regs { pub struct Identity; @@ -152,6 +160,12 @@ def item(text, marker): } mod boot { use super::*; + pub mod glue { + pub unsafe fn asahi_g15_pmp_ready() -> i32 { + super::PMP_CALLS.set(super::PMP_CALLS.get() + 1); + match super::step(13) { Ok(()) => 0, Err(e) => e } + } + } use super::{object::Objects, initdata::InitData, channel::Channels, event::Events}; pub struct Session { _owner: Owner, uat: Arc, regs: regs::Regs } impl Session { @@ -185,7 +199,9 @@ def item(text, marker): let boot::Booted { session, init, objects, channels, events, .. } = booted(); boot::Orphan { session, _objects: objects, channels: Some(channels), _init: Some(init), events } } -fn reset(fail: u32) { FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); } +fn reset(fail: u32) { + FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); PMP_CALLS.set(0); +} fn retained(label: &str) { assert_eq!(OWNER_DROPS.get(), 0, "{label}: live session owner dropped"); assert_eq!(BUFFER_DROPS.get(), 0, "{label}: firmware buffer dropped"); @@ -221,17 +237,32 @@ def item(text, marker): std::mem::forget(shared); } } - reset(0); + for orphan_oom in [false, true] { + reset((1 << 13) | if orphan_oom { 1 << 7 } else { 0 }); + let shared = Shared::new(); + let result = Firmware::start_backend(&AsahiDevice, &PMP_CONFIG, booted(), &shared); + assert!(matches!(result, Err(-113)), "PMP readiness error changed"); + assert_eq!(PMP_CALLS.get(), 1, "PMP readiness was not reported"); + retained("PMP readiness failure"); + assert_eq!(shared.orphan.lock().is_some(), !orphan_oom); + std::mem::forget(shared); + } + for cfg in [&CONFIG, &PMP_CONFIG] { + // A pending PMP error must be irrelevant on a platform without PMP. + reset(if cfg.platform == hw::Platform::Adt { 1 << 13 } else { 0 }); let shared = Shared::new(); let booted = boot::start(&shared).expect("normal boot"); assert_eq!(booted.max_frequency_khz, 1234); let uat: Arc = booted.session.uat(); - let firmware = Firmware::start_backend(&AsahiDevice, &CONFIG, booted, &shared).expect("normal backend"); + let firmware = Firmware::start_backend(&AsahiDevice, cfg, booted, &shared).expect("normal backend"); + assert_eq!(PMP_CALLS.get(), u32::from(cfg.platform == hw::Platform::AdtWithPmp), + "PMP readiness used on the wrong platform"); assert!(Arc::ptr_eq(&uat, &firmware.session.uat())); assert!(!shared.orphaned.load(Ordering::Acquire)); retained("normal success"); std::mem::forget((shared, firmware)); - println!("PASS 23 retention cases: orphan OOM, six backend failure sites, four boot-completion sites and normal success"); + } + println!("PASS 26 retention cases: orphan OOM, six backend failure sites, four boot-completion sites, PMP readiness failure and both platform success paths"); println!("normal success owner_drops={} buffer_drops={}", OWNER_DROPS.get(), BUFFER_DROPS.get()); } ''' @@ -272,11 +303,16 @@ def main(): cwd=ROOT, text=True)) variants["consume-orphan-before-allocation"] = source.replace(orphan, item(old, " pub(crate) fn orphan(")) variants["unguarded-registry-allocation"] = source.replace(backend, item(old, " fn start_backend(")) + variants["ignore-pmp-readiness-error"] = source.replace( + "let ret = unsafe { boot::glue::asahi_g15_pmp_ready() };", + "let _ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; let ret = 0;") + variants["pmp-on-other-platform"] = source.replace( + "if cfg.platform != hw::Platform::AdtWithPmp {", "if false {") receipt = {"scope": __doc__, "sources": { name: hashlib.sha256(text.encode()).hexdigest() for name, text in [("mod.rs", mod), ("boot.rs", boot), ("kbox.rs", kbox)]}, "rustc": subprocess.check_output([os.environ.get("HOSTRUSTC", "rustc"), "--version"], text=True).strip(), - "cases": 23, "public_old_source_control": CONTROL_REF, "variants": {}} + "cases": 26, "public_old_source_control": CONTROL_REF, "variants": {}} if args.mutations: receipt["old_control_sha256"] = hashlib.sha256(old.encode()).hexdigest() for name, text in variants.items(): @@ -295,14 +331,20 @@ def main(): raise RuntimeError(result.stderr) else: expected = {"consume-orphan-before-allocation": "orphan allocation: live session owner dropped", - "unguarded-registry-allocation": "backend failure: live session owner dropped"}[name] - if result.returncode == 0 or expected not in result.stderr or "left: 5" not in result.stderr: + "unguarded-registry-allocation": "backend failure: live session owner dropped", + "ignore-pmp-readiness-error": "PMP readiness error changed", + "pmp-on-other-platform": "normal backend: -113"}[name] + if result.returncode == 0 or expected not in result.stderr: raise RuntimeError("INVALID mutant outcome " + name + "\n" + result.stderr) - receipt["variants"][name]["intended_owner_drops"] = 5 + if name in ("consume-orphan-before-allocation", "unguarded-registry-allocation"): + if "left: 5" not in result.stderr: + raise RuntimeError("INVALID owner-drop count " + name) + receipt["variants"][name]["intended_owner_drops"] = 5 + receipt["variants"][name]["intended_assertion"] = expected after = subprocess.run([str(out / "actual")], capture_output=True, text=True, check=True) receipt["actual_after_mutants"] = {"exit": after.returncode, "stdout": after.stdout, "stderr": after.stderr} receipt["fixed_invariants"] = {"live_session_owner_drops": 0, "firmware_buffer_drops": 0, - "checked_cases": 23} + "checked_cases": 26} (out / "result.json").write_text(json.dumps(receipt, indent=2) + "\n") print(json.dumps(receipt, indent=2)) diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index af53ff5a9d356..f4f48aa3cafab 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -9,7 +9,8 @@ and again when the initial PB posting became a module parameter (the tuning constant PB_BLOCKS removed), and when the T6031 board source became the device tree with reference words instead of a board file (the Board enum; -the T6030 tables are unchanged). Its T6030 configuration module is +the T6030 tables are unchanged), and for publication-bound slots (removal of +MAX_VMS and its lifetime-slot comment only). Its T6030 configuration module is independently pinned to the original bytes. prepare_board() is pinned too; it was refreshed for the same Board change (its T6030 arm is unchanged). """ @@ -27,7 +28,7 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "2128be7e14bddfdc8d57951149c1fa4a276971e17bf8b6960fda0674c1fabfbe", + "hw.rs": "105395216669292954334e2bd0ecd8cd2dcb6c6faf3fc0ab95d7b5a36436b6c2", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { From e91fc1b5aee97a1267c4585cc47b77d979d3ec3a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:50:29 -0700 Subject: [PATCH 22/46] tests/asahi: model GFX report calls in PMP bridge fixtures The complete tools job found two more host fixtures that predate the T6030 PMP export changes. display26 lacks OF compatibility lookups and GFX-report service definitions; the supplier harness lacks the new report-check and report-update boundaries. Keep display26 on its modeled J613 platform and fail explicitly if it reaches a T6030 report. Model the supplier harness report service under the existing lock, exercise a forwarded report-check error, and require that power-on does not start bandwidth tracking before backend READY. Add a runtime mutant that drops the report-check error. Retain the existing configuration, supplier, readiness and serialization mutants. The production report transaction remains covered separately by the G15 PMP-vote selftest. No production code changes are made here. Generated-by: LLM Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> (cherry picked from commit 5af49598aac5ac3d6925b69a900d5417f7e84bd9) Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/display26-tests.py | 6 ++++++ tools/asahi/t6030-pmp-supplier-tests.c | 19 +++++++++++++++++++ tools/asahi/t6030-pmp-supplier-tests.py | 6 +++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tools/asahi/display26-tests.py b/tools/asahi/display26-tests.py index 8d2e497fdcf64..2366766042dc2 100644 --- a/tools/asahi/display26-tests.py +++ b/tools/asahi/display26-tests.py @@ -96,6 +96,7 @@ def main(): #include #include #include +#include #include struct device_node { int id; }; struct device { struct device_node *of_node; bool bound; int refs; }; struct device_link { int unused; }; @@ -117,12 +118,17 @@ def main(): static struct device *get_device(struct device*d){d->refs++;return d;} static void put_device(struct device*d){d->refs--;} static struct device_link *device_link_add(struct device*c,struct device*s,unsigned flags){static struct device_link l;assert(flags==1);return link_fail?NULL:&l;} +/* This harness exercises the J613/25G83 bridge, not the T6030 GFX report. */ +static bool of_machine_is_compatible(const char *name){return !strcmp(name,"apple,t8132");} +static bool of_device_is_compatible(const struct device_node*n,const char*name){return false;} ''') write(tmp, 'bridge.c', '''#include #include "drivers/soc/apple/pmp-export.c" static int ready_result,send_result,calls; int apple_pmp_report_wait_supplier_ready(struct device*d,unsigned timeout){assert(d==pmp_device);assert(timeout==5000);return ready_result;} int apple_pmp_send_power_command(const void*data,u8 command,u16 id,u32 enabled){assert(data==pmp_data);calls++;return send_result;} +int apple_pmp_report_check_t6030_gfx(struct device*d){assert(!"unexpected T6030 report on J613");return -EIO;} +int apple_pmp_report_set_t6030_gfx(struct device*d,bool enabled){assert(!"unexpected T6030 report on J613");return -EIO;} int main(void){ struct device_node n={1},wrong={2}; struct device supplier={&n,true,0},consumer={NULL,false,0}; int data; assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c index b581f3863a232..ecb6dcec9fff5 100644 --- a/tools/asahi/t6030-pmp-supplier-tests.c +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -130,6 +130,8 @@ static int pmp_lock, ready_calls, add_calls, add_result; static int poll_calls, device_locked, power_calls, power_result, link_calls; static int find_calls, lockable, present_device, link_ok; static int case_count; +/* The report transaction itself is covered by the G15 PMP-vote selftest. */ +static int gfx_check_result, gfx_check_calls, gfx_set_calls; static struct device_node *of_node_get(struct device_node *node) { @@ -375,6 +377,19 @@ static void queue_work(void *wq, int *work) managed_start.result = ready_result; managed_start.started.done = true; } +static int apple_pmp_report_check_t6030_gfx(struct device *dev) +{ + assert(dev == pmp_device && pmp_lock == 1); + gfx_check_calls++; + return gfx_check_result; +} +static int apple_pmp_report_set_t6030_gfx(struct device *dev, bool enabled) +{ + assert(dev == pmp_device && pmp_lock == 1); + (void)enabled; + gfx_set_calls++; + return 0; +} #include "t6030-pmp-production.h" static void balanced(void) @@ -457,6 +472,7 @@ static void reset(void) pmp_data = (void *)1; ready_calls = add_calls = add_result = poll_calls = 0; power_calls = power_result = link_calls = find_calls = 0; + gfx_check_result = gfx_check_calls = gfx_set_calls = 0; lockable = present_device = link_ok = 1; status_word = PMP_REPORT_READY; } @@ -498,6 +514,9 @@ int main(void) balanced(); case_count++; RUN((void)0, 0, 0, 1); + assert(gfx_check_calls == 1 && gfx_set_calls == 0); + RUN(gfx_check_result = -EINVAL, -EINVAL, 0, 0); + assert(gfx_check_calls == 1 && gfx_set_calls == 0); RUN(report14.child = &entry7; entry7.sibling = &entry17; entry17.sibling = NULL, 0, 0, 1); RUN(apple_pmp_starts.first = NULL, -ENODEV, 0, 0); diff --git a/tools/asahi/t6030-pmp-supplier-tests.py b/tools/asahi/t6030-pmp-supplier-tests.py index d323dffbef728..269006b36aa4f 100755 --- a/tools/asahi/t6030-pmp-supplier-tests.py +++ b/tools/asahi/t6030-pmp-supplier-tests.py @@ -6,7 +6,9 @@ extracted. OF enumeration/property storage, device binding, MMIO polling, and locks are controlled host boundaries. The T6030 managed entry wait and start wait are extracted too; the firmware/workqueue boundary is controlled. -This models ownership/error handling, not real teardown concurrency or DMA. +The GFX-report service is modeled separately from supplier readiness; its +transaction has its own production-body PMP-vote selftest. This models +ownership/error handling, not real teardown concurrency or DMA. """ import argparse import hashlib @@ -139,6 +141,8 @@ def main(): ("gate-config", "IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && gate_soc == &gate_t6030", "gate_soc == &gate_t6030"), ("gate-phandle-shape", 'of_property_count_u32_elems(gpu, "apple,pmp") != 1', "false"), ("serialized-command", "\tguard(mutex)(&pmp_lock);", "\t(void)&pmp_lock;"), + ("gfx-check-error", "ret = apple_pmp_report_check_t6030_gfx(pmp_device);", + "ret = (apple_pmp_report_check_t6030_gfx(pmp_device), 0);"), ] for name, old, new in mutations: assert code.count(old) == 1, (name, code.count(old)) From 6ec4d35fc5c4f7e872eec94cbf8bb64edacbefdf Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:08:12 -0700 Subject: [PATCH 23/46] drm/asahi: g15: start T6031 after its native display, then autoload it T6031 was kept out of the G15 autoload table until it had an admission gate ordering its first firmware start after the native display, as T6030 has through its display and PMP links. Give drm/apple one ready/defer helper with a publication slot per SoC display subsystem, published after DRM registration and client setup and withdrawn before unbind, and export apple_drm_t6031_link_device() beside the T6030 one. T6031 admission calls it after the J516C machine check and before anything else: it defers until that display is published, so a boot that leaves the display disabled keeps the GPU waiting (logged once) rather than starting it. T6030 is unchanged and T6031 gains no PMP vote. Then add apple,agx-t6031 to the alias table. A manual load on J516C now waits for the native display as well. The GPU still needs the m1n1 GPU handoff, the opt-in native display (apple_t6031_display.enable=1) and the pinned board file; this change provides none of them. tools/asahi/g15-display-order-tests.py runs the actual bodies on host models with mutants and is registered in tools/aurora-ci/host-tests.json. Generated-by: LLM Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 8ea9a7235304bb9ad612a22db461c798e0f8d6ec) [ryan: use the Linux 7.3 of_device_table! form without ID info for the autoload table; register the test in tools/asahi/run-host-tests.py instead of omacom's tools/aurora-ci/host-tests.json] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 71 +++-- drivers/gpu/drm/asahi/g15/boot.rs | 12 + drivers/gpu/drm/asahi/g15/glue.c | 23 ++ drivers/gpu/drm/asahi/g15/mod.rs | 16 +- include/linux/soc/apple/display.h | 1 + tools/asahi/g15-display-order-tests.py | 409 +++++++++++++++++++++++++ tools/asahi/run-host-tests.py | 1 + 7 files changed, 505 insertions(+), 28 deletions(-) create mode 100644 tools/asahi/g15-display-order-tests.py diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d71c5c0370f2e..007ce2913c158 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -64,28 +64,44 @@ struct apple_drm_private { }; /* Component readiness is later than platform probe/bind. */ -static DEFINE_MUTEX(apple_t6030_display_lock); +static DEFINE_MUTEX(apple_g15_display_lock); static struct device *apple_t6030_display; +static struct device *apple_t6031_display; -static int apple_t6030_display_publish(struct device *dev) +/* The publication slot of a G15 SoC's internal display subsystem, or NULL. */ +static struct device **apple_g15_display_slot(struct device *dev) { - if (!of_device_is_compatible(dev->of_node, "apple,t6030-display-subsystem")) + if (of_device_is_compatible(dev->of_node, "apple,t6030-display-subsystem")) + return &apple_t6030_display; + if (of_device_is_compatible(dev->of_node, "apple,t6031-display-subsystem")) + return &apple_t6031_display; + return NULL; +} + +static int apple_g15_display_publish(struct device *dev) +{ + struct device **slot = apple_g15_display_slot(dev); + + if (!slot) return 0; - guard(mutex)(&apple_t6030_display_lock); - if (apple_t6030_display) - return apple_t6030_display == dev ? 0 : -EBUSY; - apple_t6030_display = get_device(dev); + guard(mutex)(&apple_g15_display_lock); + if (*slot) + return *slot == dev ? 0 : -EBUSY; + *slot = get_device(dev); return 0; } -static void apple_t6030_display_withdraw(struct device *dev) +static void apple_g15_display_withdraw(struct device *dev) { + struct device **slot = apple_g15_display_slot(dev); struct device *old = NULL; - scoped_guard(mutex, &apple_t6030_display_lock) { - if (apple_t6030_display == dev) { - old = apple_t6030_display; - apple_t6030_display = NULL; + if (!slot) + return; + scoped_guard(mutex, &apple_g15_display_lock) { + if (*slot == dev) { + old = *slot; + *slot = NULL; } } put_device(old); @@ -96,22 +112,35 @@ static void apple_t6030_display_withdraw(struct device *dev) * publish separately. This lock protects only publication and a device link; * no display calls or firmware RPC run while it is held. */ -int apple_drm_t6030_link_device(struct device *consumer) +static int apple_g15_link_device(struct device *consumer, const char *machine, + const char *gpu, struct device **display) { struct device_link *link; - if (!of_machine_is_compatible("apple,t6030") || !consumer->of_node || - !of_device_is_compatible(consumer->of_node, "apple,agx-t6030")) + if (!of_machine_is_compatible(machine) || !consumer->of_node || + !of_device_is_compatible(consumer->of_node, gpu)) return -ENODEV; - guard(mutex)(&apple_t6030_display_lock); - if (!apple_t6030_display) + guard(mutex)(&apple_g15_display_lock); + if (!*display) return -EPROBE_DEFER; - link = device_link_add(consumer, apple_t6030_display, - DL_FLAG_AUTOREMOVE_CONSUMER); + link = device_link_add(consumer, *display, DL_FLAG_AUTOREMOVE_CONSUMER); return link ? 0 : -EINVAL; } + +int apple_drm_t6030_link_device(struct device *consumer) +{ + return apple_g15_link_device(consumer, "apple,t6030", "apple,agx-t6030", + &apple_t6030_display); +} EXPORT_SYMBOL_GPL(apple_drm_t6030_link_device); +int apple_drm_t6031_link_device(struct device *consumer) +{ + return apple_g15_link_device(consumer, "apple,t6031", "apple,agx-t6031", + &apple_t6031_display); +} +EXPORT_SYMBOL_GPL(apple_drm_t6031_link_device); + DEFINE_DRM_GEM_DMA_FOPS(apple_fops); #define DART_PAGE_SIZE 16384 @@ -1316,7 +1345,7 @@ static int apple_drm_init(struct device *dev) drm_client_setup_with_fourcc(&apple->drm, DRM_FORMAT_XRGB8888); /* A subsequent successful platform probe retries deferred GPU probes. */ - ret = apple_t6030_display_publish(dev); + ret = apple_g15_display_publish(dev); if (ret) { drm_dev_unregister(&apple->drm); goto err_unbind; @@ -1338,7 +1367,7 @@ static void apple_drm_uninit(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - apple_t6030_display_withdraw(dev); + apple_g15_display_withdraw(dev); /* An aborted takeover leaves its DCPs running until the reboot. */ if (apple->aborted) diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index 388391ec311f7..70decd06fca0d 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -72,6 +72,7 @@ pub(crate) mod glue { fn asahi_g15_ioremap_wc(start: u64, size: usize) -> *mut core::ffi::c_void; fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; pub(crate) fn asahi_g15_pmp_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + pub(crate) fn asahi_g15_display_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_get(dev: *mut core::ffi::c_void) -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_ready() -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_put(dev: *mut core::ffi::c_void); @@ -486,6 +487,17 @@ pub(crate) fn admit( } else if !glue::machine(cfg.machine) { dev_err!(dev, "G15: not a {} machine\n", cfg.name); return Err(ENODEV); + } else { + // No PMP vote here, but the first start follows the native display all the same. + // SAFETY: `dev` is the bound GPU device. + let ret = unsafe { glue::asahi_g15_display_prepare(dev.as_raw().cast()) }; + if ret != 0 { + let err = Error::from_errno(ret); + if err != EPROBE_DEFER { + dev_err!(dev, "native display not admitted: {:?}\n", err); + } + return Err(err); + } } let adt = adt::read(dev, cfg) .inspect_err(|e| dev_err!(dev, "GPU regions not admitted: {:?}\n", e))?; diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index 8d6f3658322df..adda36d4bac26 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -37,6 +37,7 @@ unsigned int asahi_g15_memory_mode(u64 start, u64 size); void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size); bool asahi_g15_dma_coherent(struct device *dev); int asahi_g15_pmp_prepare(struct device *dev); +int asahi_g15_display_prepare(struct device *dev); int asahi_g15_pmp_get(struct device *dev); int asahi_g15_pmp_ready(void); void asahi_g15_pmp_put(struct device *dev); @@ -145,6 +146,28 @@ int asahi_g15_pmp_prepare(struct device *dev) return apple_pmp_link_device(dev); } +/* + * T6031 has no PMP vote, but its first firmware start follows the native + * display as T6030's does: defer until that display is up. Its display is + * opt-in (apple_t6031_display.enable=1), so say once why the GPU waits. + */ +int asahi_g15_display_prepare(struct device *dev) +{ + struct device_node *np; + int ret; + + if (!dev->of_node || !of_device_is_compatible(dev->of_node, "apple,agx-t6031")) + return -ENODEV; + ret = apple_drm_t6031_link_device(dev); + if (ret != -EPROBE_DEFER) + return ret; + np = of_find_compatible_node(NULL, NULL, "apple,t6031-display-subsystem"); + if (!np || !of_device_is_available(np)) + dev_info_once(dev, "G15: waiting for the T6031 native display, which this boot did not enable (apple_t6031_display.enable=1 and the boot loader's display handoff)\n"); + of_node_put(np); + return ret; +} + int asahi_g15_pmp_get(struct device *dev) { if (!asahi_g15_board() || !dev->of_node || diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index fa5ed4fbaa333..9ffd858598b1c 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait -//! for the native display and PMP supplier before booting firmware. T6031 retains explicit -//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for +//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting +//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -152,13 +152,15 @@ const OF_TABLE: kernel::device_id::IdArray = ), ]); -// T6030 probe admission requires the native display and PMP supplier before -// firmware starts. T6031 retains explicit loading until it has an equivalent -// display-order admission gate. +// Probe admission requires the native display before firmware starts (T6030 +// also its PMP supplier), so both may be loaded at device creation. kernel::of_device_table!( AUTOLOAD_OF_TABLE, @none, - [of::DeviceId::new(c_str!("apple,agx-t6030"))] + [ + of::DeviceId::new(c_str!("apple,agx-t6030")), + of::DeviceId::new(c_str!("apple,agx-t6031")), + ] ); /// The G15 platform adapter: the shared DRM registration probed through diff --git a/include/linux/soc/apple/display.h b/include/linux/soc/apple/display.h index 8978d95d40484..eb71ce91e0919 100644 --- a/include/linux/soc/apple/display.h +++ b/include/linux/soc/apple/display.h @@ -6,5 +6,6 @@ struct device; /* Initial native display admission, before starting retained G15 firmware. */ int apple_drm_t6030_link_device(struct device *consumer); +int apple_drm_t6031_link_device(struct device *consumer); #endif diff --git a/tools/asahi/g15-display-order-tests.py b/tools/asahi/g15-display-order-tests.py new file mode 100644 index 0000000000000..2fd7aa99a2bbe --- /dev/null +++ b/tools/asahi/g15-display-order-tests.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the actual G15 native-display start-order bodies on host models. + +The publication slots, the shared link helper and both SoC wrappers are taken +from drivers/gpu/drm/apple/apple_drv.c, the T6031 glue from g15/glue.c and +admit() from g15/boot.rs, unchanged. OF lookups, device references, links and +the admission steps are host models; no device or firmware is touched. Each +mutant must build and fail an intended assertion. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +APPLE = "drivers/gpu/drm/apple/apple_drv.c" +GLUE = "drivers/gpu/drm/asahi/g15/glue.c" +BOOT = "drivers/gpu/drm/asahi/g15/boot.rs" +MODULE = "drivers/gpu/drm/asahi/g15/mod.rs" + + +def item(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +def c_source(root): + apple = (root / APPLE).read_text() + glue = (root / GLUE).read_text() + state = [line for line in apple.splitlines() + if line in ("static DEFINE_MUTEX(apple_g15_display_lock);", + "static struct device *apple_t6030_display;", + "static struct device *apple_t6031_display;")] + assert len(state) == 3, f"publication state not found: {state}" + bodies = [item(apple, marker) for marker in ( + "\nstatic struct device **apple_g15_display_slot(struct device *dev)\n{", + "\nstatic int apple_g15_display_publish(struct device *dev)\n{", + "\nstatic void apple_g15_display_withdraw(struct device *dev)\n{", + "\nstatic int apple_g15_link_device(", + "\nint apple_drm_t6030_link_device(struct device *consumer)\n{", + "\nint apple_drm_t6031_link_device(struct device *consumer)\n{", + )] + bodies.append(item(glue, "\nint asahi_g15_display_prepare(struct device *dev)\n{")) + return "\n".join(state) + "\n\n" + "\n\n".join(bodies) + "\n" + + +C_PREFIX = r''' +#include +#include +#include +#include +#include + +#define ENODEV 19 +#define EBUSY 16 +#define EINVAL 22 +#define EPROBE_DEFER 517 +#define DL_FLAG_AUTOREMOVE_CONSUMER 1u +#define EXPORT_SYMBOL_GPL(sym) extern int sym##_exported + +struct device_node { const char *compatible; bool available; int refs; }; +struct device { struct device_node *of_node; int refs; }; +struct device_link { struct device *consumer, *supplier; unsigned int flags; }; +struct mutex { bool held; }; + +static const char *machine_compat[2]; +static struct device_node *nodes[4]; +static int links_made, link_fail, infos; +static struct device_link last_link; + +static int of_machine_is_compatible(const char *compat) +{ + for (int i = 0; i < 2; i++) + if (machine_compat[i] && !strcmp(machine_compat[i], compat)) + return 1; + return 0; +} +static int of_device_is_compatible(const struct device_node *np, const char *compat) +{ + return np && !strcmp(np->compatible, compat); +} +static struct device_node *of_find_compatible_node(struct device_node *from, const char *type, + const char *compat) +{ + (void)from; (void)type; + for (int i = 0; i < 4; i++) + if (nodes[i] && !strcmp(nodes[i]->compatible, compat)) { + nodes[i]->refs++; + return nodes[i]; + } + return NULL; +} +static bool of_device_is_available(const struct device_node *np) { return np && np->available; } +static void of_node_put(struct device_node *np) { if (np) np->refs--; } +static struct device *get_device(struct device *dev) { dev->refs++; return dev; } +static void put_device(struct device *dev) { if (dev) dev->refs--; } +static struct device_link *device_link_add(struct device *consumer, struct device *supplier, + unsigned int flags) +{ + if (link_fail) + return NULL; + links_made++; + last_link = (struct device_link){ consumer, supplier, flags }; + return &last_link; +} +#define dev_info_once(dev, ...) do { static bool once_; (void)(dev); if (!once_) { once_ = true; infos++; } } while (0) + +static void unlock_(struct mutex **m) { if (*m) { assert((*m)->held); (*m)->held = false; } } +static struct mutex *lock_(struct mutex *m) { assert(!m->held); m->held = true; return m; } +#define DEFINE_MUTEX(name) struct mutex name = { false } +#define guard(type) struct mutex *guard_ __attribute__((cleanup(unlock_))) = lock_ +#define scoped_guard(type, m) for (struct mutex *sg_ __attribute__((cleanup(unlock_))) = lock_(m), *done_ = NULL; !done_; done_ = (struct mutex *)1) +''' + +C_MAIN = r''' +static const char *scenario; +#define CHECK(c) do { if (!(c)) { fprintf(stderr, "%s:%d: %s\n", scenario, __LINE__, #c); exit(1); } } while (0) + +static struct device_node gpu31 = { "apple,agx-t6031", true }, gpu30 = { "apple,agx-t6030", true }; +static struct device_node disp31 = { "apple,t6031-display-subsystem", false }; +static struct device_node disp30 = { "apple,t6030-display-subsystem", true }; +static struct device_node other = { "apple,display-subsystem", true }; +static struct device g31 = { &gpu31 }, g30 = { &gpu30 }, none = { NULL }; +static struct device d31 = { &disp31 }, d31b = { &disp31 }, d30 = { &disp30 }, dother = { &other }; + +static void machine(const char *soc, const char *board) +{ + machine_compat[0] = board; + machine_compat[1] = soc; +} +static void clean(void) +{ + CHECK(!apple_g15_display_lock.held); + for (int i = 0; i < 4; i++) + CHECK(!nodes[i] || nodes[i]->refs == 0); +} +static int prepare(struct device *dev) { int ret = asahi_g15_display_prepare(dev); clean(); return ret; } +static int link30(struct device *dev) { int ret = apple_drm_t6030_link_device(dev); clean(); return ret; } +static int link31(struct device *dev) { int ret = apple_drm_t6031_link_device(dev); clean(); return ret; } +static int publish(struct device *dev) { int ret = apple_g15_display_publish(dev); clean(); return ret; } +static void withdraw(struct device *dev) { apple_g15_display_withdraw(dev); clean(); } + +int main(int argc, char **argv) +{ + scenario = argv[1]; + machine("apple,t6031", "apple,j516c"); + nodes[0] = &gpu31; + nodes[1] = &disp31; + nodes[2] = &disp30; + nodes[3] = &other; + if (!strcmp(scenario, "disabled")) { + /* The gate left the display disabled: no start, no success, said once. */ + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1); + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + } else if (!strcmp(scenario, "absent")) { + nodes[1] = NULL; + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + } else if (!strcmp(scenario, "pending")) { + disp31.available = true; + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 0 && links_made == 0); + } else if (!strcmp(scenario, "ready")) { + disp31.available = true; + CHECK(publish(&d31) == 0 && d31.refs == 1); + CHECK(prepare(&g31) == 0 && links_made == 1 && infos == 0); + CHECK(last_link.consumer == &g31 && last_link.supplier == &d31 && + last_link.flags == DL_FLAG_AUTOREMOVE_CONSUMER); + link_fail = 1; + CHECK(prepare(&g31) == -EINVAL); + link_fail = 0; + CHECK(publish(&d31) == 0 && d31.refs == 1); + CHECK(publish(&d31b) == -EBUSY && d31b.refs == 0); + withdraw(&d31b); + CHECK(prepare(&g31) == 0); + withdraw(&d31); + CHECK(d31.refs == 0 && prepare(&g31) == -EPROBE_DEFER); + } else if (!strcmp(scenario, "isolation")) { + disp31.available = true; + CHECK(publish(&d30) == 0 && d30.refs == 1); + CHECK(publish(&dother) == 0 && dother.refs == 0); + withdraw(&dother); + CHECK(prepare(&g31) == -EPROBE_DEFER && links_made == 0); + } else if (!strcmp(scenario, "foreign")) { + disp31.available = true; + CHECK(publish(&d31) == 0); + CHECK(prepare(&g30) == -ENODEV && link31(&g30) == -ENODEV); + CHECK(prepare(&none) == -ENODEV && link31(&none) == -ENODEV); + machine("apple,t6030", "apple,j516s"); + CHECK(link31(&g31) == -ENODEV && links_made == 0); + } else if (!strcmp(scenario, "t6030")) { + machine("apple,t6030", "apple,j516s"); + CHECK(link30(&g30) == -EPROBE_DEFER); + disp31.available = true; + CHECK(publish(&d31) == 0 && link30(&g30) == -EPROBE_DEFER); + CHECK(publish(&d30) == 0 && link30(&g30) == 0 && last_link.supplier == &d30); + withdraw(&d30); + CHECK(d30.refs == 0 && link30(&g30) == -EPROBE_DEFER); + CHECK(link30(&g31) == -ENODEV); + } else { + CHECK(!"unknown scenario"); + } + return 0; +} +''' + +C_SCENARIOS = ["disabled", "absent", "pending", "ready", "isolation", "foreign", "t6030"] + +C_MUTANTS = { + "unpublished-succeeds": ("if (!*display)\n\t\treturn -EPROBE_DEFER;", "if (!*display)\n\t\treturn 0;"), + "disabled-display-skipped": (re.compile(r"\t\tdev_info_once\(dev,[^;]*;\n"), "\t\t{ of_node_put(np); return 0; }\n"), + "t6031-reads-t6030-slot": ("&apple_t6031_display);", "&apple_t6030_display);"), + "t6031-published-as-t6030": ("return &apple_t6031_display;", "return &apple_t6030_display;"), + "no-machine-check": ("!of_machine_is_compatible(machine) ||", ""), + "withdraw-keeps-display": ("*slot = NULL;", ";"), + "publish-without-reference": ("*slot = get_device(dev);", "*slot = dev;"), +} + +RS_PREFIX = r''' +#![allow(dead_code, unused_variables, unused_imports, non_snake_case)] +use std::cell::RefCell; +use std::marker::PhantomData; +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +struct Error(i32); +impl Error { fn from_errno(errno: i32) -> Self { Self(errno) } } +type Result = std::result::Result; +const ENODEV: Error = Error(-19); +const EINVAL: Error = Error(-22); +const EPROBE_DEFER: Error = Error(-517); +macro_rules! dev_err { ($($args:tt)*) => {{}}; } +macro_rules! dev_info { ($($args:tt)*) => {{}}; } +struct State { board: bool, machine: bool, pmp: i32, display: i32 } +thread_local! { + static CALLS: RefCell> = const { RefCell::new(Vec::new()) }; + static STATE: RefCell = const { RefCell::new(State { board: true, machine: true, pmp: 0, display: 0 }) }; +} +fn call(name: &'static str) { CALLS.with(|c| c.borrow_mut().push(name)); } +mod device { + pub struct Device; + impl Device { pub fn as_raw(&self) -> *mut u8 { std::ptr::null_mut() } } +} +mod platform { + pub struct Device(pub super::device::Device, pub C); + impl Device { pub fn as_ref(&self) -> &super::device::Device { &self.0 } } +} +struct Core<'a>(PhantomData<&'a ()>); +struct AsahiDevice; +mod hw { + #[derive(Copy, Clone, PartialEq, Eq)] pub enum Platform { AdtWithPmp, DeviceTree } + #[derive(Copy, Clone, PartialEq, Eq)] pub enum Board { None, Templates, Blob } +} +struct Config { platform: hw::Platform, machine: &'static str, name: &'static str, firmware: (), board: hw::Board } +mod adt { + pub struct GpuAdt; + pub fn read(_: &super::device::Device, _: &super::Config) -> super::Result { super::call("adt"); Ok(GpuAdt) } +} +mod identity { + pub fn firmware(_: &super::device::Device, _: &super::adt::GpuAdt, _: ()) -> super::Result { super::call("identity"); Ok(()) } +} +mod initdata { + pub fn prepare_board(_: &super::device::Device, _: &super::Config, _: &super::adt::GpuAdt) -> super::Result { + super::call("board_images"); Ok(1) + } +} +mod regs { + pub struct Regs; + impl Regs { + pub fn new(_: &super::platform::Device) -> super::Result { super::call("regs"); Ok(Self) } + pub fn asc_control(&self) -> super::Result { Ok(0) } + } +} +mod glue { + use super::{call, STATE}; + pub fn board() -> bool { call("board"); STATE.with(|s| s.borrow().board) } + pub fn machine(_: &str) -> bool { call("machine"); STATE.with(|s| s.borrow().machine) } + pub unsafe fn asahi_g15_pmp_prepare(_: *mut core::ffi::c_void) -> core::ffi::c_int { call("pmp"); STATE.with(|s| s.borrow().pmp) } + pub unsafe fn asahi_g15_display_prepare(_: *mut core::ffi::c_void) -> core::ffi::c_int { call("display"); STATE.with(|s| s.borrow().display) } + pub fn dma_coherent(_: &super::device::Device) -> bool { call("coherent"); false } + pub fn hold_power(_: &super::device::Device) { call("hold"); } +} +struct Admitted { regs: regs::Regs, adt: adt::GpuAdt, board_images: u32 } +''' + +RS_MAIN = r''' +static T6031: Config = Config { platform: hw::Platform::DeviceTree, machine: "apple,j516c", name: "T6031 G15C", firmware: (), board: hw::Board::Blob }; +static T6030: Config = Config { platform: hw::Platform::AdtWithPmp, machine: "apple,t6030", name: "T6030 G15S", firmware: (), board: hw::Board::Templates }; +const TAIL: [&str; 6] = ["adt", "identity", "coherent", "board_images", "regs", "hold"]; +fn run(cfg: &'static Config, machine: bool, pmp: i32, display: i32) -> (Result<()>, Vec<&'static str>) { + CALLS.with(|c| c.borrow_mut().clear()); + STATE.with(|s| *s.borrow_mut() = State { board: true, machine, pmp, display }); + let pdev = platform::Device(device::Device, Core(PhantomData)); + let ret = admit(&pdev, &AsahiDevice, cfg).map(|_| ()); + (ret, CALLS.with(|c| c.borrow().clone())) +} +fn with_tail(head: &[&'static str]) -> Vec<&'static str> { head.iter().chain(TAIL.iter()).copied().collect() } +fn main() { + assert_eq!(run(&T6031, true, 0, -517), (Err(EPROBE_DEFER), vec!["machine", "display"]), "T6031 defers before firmware"); + assert_eq!(run(&T6031, true, 0, -19), (Err(ENODEV), vec!["machine", "display"]), "T6031 refusal stops admission"); + assert_eq!(run(&T6031, true, 0, 0), (Ok(()), with_tail(&["machine", "display"])), "T6031 admitted after its display"); + assert_eq!(run(&T6031, false, 0, 0), (Err(ENODEV), vec!["machine"]), "the machine check comes first"); + assert_eq!(run(&T6030, true, 0, -517), (Ok(()), with_tail(&["board", "pmp"])), "T6030 path unchanged"); + assert_eq!(run(&T6030, true, -517, 0), (Err(EPROBE_DEFER), vec!["board", "pmp"]), "T6030 still defers on its PMP link"); +} +''' + +RS_MUTANTS = { + "display-refusal-ignored": ("dev_err!(dev, \"native display not admitted: {:?}\\n\", err);\n }\n return Err(err);", + "dev_err!(dev, \"native display not admitted: {:?}\\n\", err);\n }"), + "display-order-skipped": ("let ret = unsafe { glue::asahi_g15_display_prepare(dev.as_raw().cast()) };", "let ret = 0;"), +} + + +def mutate(source, change): + old, new = change + if isinstance(old, re.Pattern): + mutated, count = old.subn(new, source, count=1) + else: + count = source.count(old) + mutated = source.replace(old, new, 1) + assert count >= 1, f"mutation anchor not found: {old!r}" + return mutated + + +def compile_c(work, source, name): + src = work / f"{name}.c" + exe = work / name + src.write_text(C_PREFIX + source + C_MAIN) + result = subprocess.run(["gcc", "-std=gnu11", "-Wall", "-Werror", "-Wno-unused-function", + "-o", str(exe), str(src)], capture_output=True, text=True) + if result.returncode: + raise SystemExit(f"{name}: does not build:\n{result.stderr}") + return exe + + +def c_failures(exe): + return [s for s in C_SCENARIOS if subprocess.run([str(exe), s], capture_output=True).returncode] + + +def compile_rs(work, rustc, source, name): + src = work / f"{name}.rs" + exe = work / name + src.write_text(RS_PREFIX + source + RS_MAIN) + result = subprocess.run([rustc, "--edition=2021", "-o", str(exe), str(src)], capture_output=True, text=True) + if result.returncode: + raise SystemExit(f"{name}: does not build with {rustc}:\n{result.stderr}") + return exe + + +def static_checks(root): + apple = (root / APPLE).read_text() + module = (root / MODULE).read_text() + glue = (root / GLUE).read_text() + compat = re.compile(r'of::DeviceId::new\(c_str!\("([^"]+)"\)\)') + start = module.index("kernel::of_device_table!(\n AUTOLOAD_OF_TABLE,") + autoload = module[start:module.index("\n);", start)] + start = module.index("const OF_TABLE:") + matched = module[start:module.index("]);", start)] + assert compat.findall(autoload) == ["apple,agx-t6030", "apple,agx-t6031"], "autoload table" + assert compat.findall(matched) == ["apple,agx-t6030", "apple,agx-t6031"], "match table" + init = item(apple, "\nstatic int apple_drm_init(struct device *dev)\n{") + order = [init.index(s) for s in ("drm_dev_register(", "drm_client_setup_with_fourcc(", "apple_g15_display_publish(dev)")] + assert order == sorted(order), "publication must follow DRM registration and client setup" + uninit = item(apple, "\nstatic void apple_drm_uninit(struct device *dev)\n{") + assert uninit.index("apple_g15_display_withdraw(dev)") < uninit.index("component_unbind_all("), "withdraw before unbind" + assert "apple_t6030_display_publish" not in apple and "apple_t6030_display_withdraw" not in apple + pmp = item(glue, "\nint asahi_g15_pmp_prepare(struct device *dev)\n{") + assert pmp.index("apple_drm_t6030_link_device(dev)") < pmp.index("apple_pmp_link_device(dev)"), "T6030 order" + return 5 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--rustc", action="append", default=[], help="rustc to build the admit() model with") + args = parser.parse_args() + rustcs = args.rustc or ["rustc"] + checks = static_checks(args.root) + source = c_source(args.root) + admit = item((args.root / BOOT).read_text(), "pub(crate) fn admit(") + with tempfile.TemporaryDirectory(prefix="g15-display-order-", dir=os.environ.get("TMPDIR")) as tmp: + work = Path(tmp) + failed = c_failures(compile_c(work, source, "control")) + assert not failed, f"control scenarios failed: {failed}" + for name, change in C_MUTANTS.items(): + failed = c_failures(compile_c(work, mutate(source, change), f"mutant-{name}")) + assert failed, f"C mutant {name} survived" + print(f"mutant {name}: caught by {', '.join(failed)}") + for rustc in rustcs: + exe = compile_rs(work, rustc, admit, "admit") + run = subprocess.run([str(exe)], capture_output=True, text=True) + assert run.returncode == 0, f"admit() control failed with {rustc}:\n{run.stderr}" + for name, change in RS_MUTANTS.items(): + exe = compile_rs(work, rustc, mutate(admit, change), f"admit-{name}") + run = subprocess.run([str(exe)], capture_output=True, text=True) + assert run.returncode, f"admit() mutant {name} survived with {rustc}" + print(f"mutant {name} ({rustc}): caught") + print(f"PASS: {checks} static checks, {len(C_SCENARIOS)} C scenarios, " + f"{len(C_MUTANTS)} C mutants, admit() on {len(rustcs)} rustc, {len(RS_MUTANTS)} admit() mutants") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 21e1f7f1f8fbe..15f52ce90ffb6 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -92,6 +92,7 @@ def mutations(flag, names, base, expect=KILL): 'display26-tests.py': [([], PASS)], # The old revision publishes the fd before copy_to_user() succeeds. 'dma-heap-fd-tests.py': [(out() + ['--old-ref', 'cb732d027aa18e1fcf9d2797f47d20b179ebc59c'], PASS)], + 'g15-display-order-tests.py': [([], PASS)], 'g15-input-error-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS), (out('--output') + ['--ref', 'ede99b3c1990b4a4a8c045fdf627a2bd0f14446a'], PASS)], 'g15-maintained-handoff-tests.py': [(out('--output'), PASS), (out('--output') + ['--mutations'], PASS)], From 50308e74f6c17f933e420e9c8ee252b87faa34ba Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:47:43 -0700 Subject: [PATCH 24/46] tools/asahi: t6030 display admission tests: follow the shared display slots The T6030 display publisher and link now go through the shared G15 display slots and link helper (8ea9a723). This suite's anchors and C fixture still named the removed apple_t6030_display_publish/withdraw and their lock, so it failed at its first source-order check. Extract the slot, publish and withdraw functions, the shared link helper and the T6030 wrapper, and give the fixture the shared lock and the T6031 slot. The unrelated-display control now uses a non-G15 display subsystem. A new case checks that a published T6031 display takes its own slot and never satisfies T6030 admission. The seven mutants are carried over to the shared code; wrong-gpu keeps the helper's parameter used so that it still builds. One more mutant maps the T6031 slot onto the T6030 one. Every mutant builds and is killed. Production code is unchanged. Generated-by: LLM Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c69f2b118865d319f61ed453b23de6cd12a1ada6) Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/t6030-display-admission-tests.c | 38 +++++++++++++------- tools/asahi/t6030-display-admission-tests.py | 29 ++++++++------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/tools/asahi/t6030-display-admission-tests.c b/tools/asahi/t6030-display-admission-tests.c index e2c75eddb5ea5..ebf3e2e868324 100644 --- a/tools/asahi/t6030-display-admission-tests.c +++ b/tools/asahi/t6030-display-admission-tests.c @@ -22,12 +22,14 @@ struct device { struct device_link { int unused; }; -static pthread_mutex_t apple_t6030_display_lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t apple_g15_display_lock = PTHREAD_MUTEX_INITIALIZER; static struct device *apple_t6030_display; +static struct device *apple_t6031_display; static struct device_node display_node = { "apple,t6030-display-subsystem" }, gpu_node = { "apple,agx-t6030" }, - other_node = { "apple,t6031-display-subsystem" }; -static struct device display, duplicate, gpu; + other_node = { "apple,display-subsystem" }, + t6031_display_node = { "apple,t6031-display-subsystem" }; +static struct device display, duplicate, gpu, t6031_display; static const char *machine = "apple,t6030"; static int pmp_calls, link_calls, pmp_result; static bool link_ok = true; @@ -106,7 +108,7 @@ static int apple_pmp_link_device(struct device *dev) #include "production.h" static void reset(void) { - assert(!apple_t6030_display && !locked); + assert(!apple_t6030_display && !apple_t6031_display && !locked); display = (struct device){ .of_node = &display_node, .refs = 1 }; duplicate = (struct device){ .of_node = &display_node, .refs = 1 }; gpu = (struct device){ .of_node = &gpu_node, .refs = 1 }; @@ -127,7 +129,7 @@ static void *admit_thread(void *unused) static void *withdraw_thread(void *unused) { (void)unused; - apple_t6030_display_withdraw(&display); + apple_g15_display_withdraw(&display); atomic_store(&withdrew, true); return NULL; } @@ -140,18 +142,18 @@ int main(void) assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && !pmp_calls); cases++; /* Component platform bind alone never calls this publication. Retry succeeds only afterward. */ - assert(apple_t6030_display_publish(&display) == 0 && + assert(apple_g15_display_publish(&display) == 0 && atomic_load(&display.refs) == 2); cases++; assert(asahi_g15_pmp_prepare(&gpu) == 0 && pmp_calls == 1); cases++; - assert(apple_t6030_display_publish(&display) == 0 && + assert(apple_g15_display_publish(&display) == 0 && atomic_load(&display.refs) == 2); cases++; - assert(apple_t6030_display_publish(&duplicate) == -EBUSY && + assert(apple_g15_display_publish(&duplicate) == -EBUSY && apple_t6030_display == &display); cases++; - apple_t6030_display_withdraw(&duplicate); + apple_g15_display_withdraw(&duplicate); assert(apple_t6030_display == &display && atomic_load(&display.refs) == 2); cases++; @@ -170,17 +172,27 @@ int main(void) assert(apple_drm_t6030_link_device(&gpu) == -ENODEV); cases++; gpu.of_node = &gpu_node; - apple_t6030_display_withdraw(&display); + apple_g15_display_withdraw(&display); assert(atomic_load(&display.refs) == 1 && !apple_t6030_display); cases++; assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && pmp_calls == 2); cases++; display.of_node = &other_node; - assert(apple_t6030_display_publish(&display) == 0 && - !apple_t6030_display && atomic_load(&display.refs) == 1); + assert(apple_g15_display_publish(&display) == 0 && + !apple_t6030_display && !apple_t6031_display && + atomic_load(&display.refs) == 1); + cases++; + /* The T6031 display has its own slot: it never satisfies T6030 admission. */ + t6031_display = (struct device){ .of_node = &t6031_display_node, .refs = 1 }; + assert(apple_g15_display_publish(&t6031_display) == 0 && + apple_t6031_display == &t6031_display && !apple_t6030_display && + atomic_load(&t6031_display.refs) == 2); + assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && pmp_calls == 2); + apple_g15_display_withdraw(&t6031_display); + assert(!apple_t6031_display && atomic_load(&t6031_display.refs) == 1); cases++; reset(); - assert(!apple_t6030_display_publish(&display)); + assert(!apple_g15_display_publish(&display)); atomic_store(&pause_link, true); pthread_t admit, withdraw; assert(!pthread_create(&admit, NULL, admit_thread, NULL)); diff --git a/tools/asahi/t6030-display-admission-tests.py b/tools/asahi/t6030-display-admission-tests.py index a1da987377f18..479c11b00fed0 100644 --- a/tools/asahi/t6030-display-admission-tests.py +++ b/tools/asahi/t6030-display-admission-tests.py @@ -2,9 +2,11 @@ # SPDX-License-Identifier: GPL-2.0-only """Actual T6030 display publisher and G15 admission, with pthread boundaries. -Tests initial admission and pointer lifetime, not safe shutdown of retained G15 -firmware after forced display unbind. Full DRM publication placement is checked -as source ordering separately from executable host boundary controls. +The publisher is the shared G15 display slot code; T6030 admission goes through +the shared link helper. Tests initial admission and pointer lifetime, not safe +shutdown of retained G15 firmware after forced display unbind. Full DRM +publication placement is checked as source ordering separately from executable +host boundary controls. """ import argparse import hashlib @@ -26,23 +28,26 @@ def main(): glue = (ROOT / 'drivers/gpu/drm/asahi/g15/glue.c').read_text() init = extract.function(drv, 'static int apple_drm_init(') uninit = extract.function(drv, 'static void apple_drm_uninit(') - assert init.index('ret = drm_dev_register(') < init.index('drm_client_setup_with_fourcc(') < init.index('ret = apple_t6030_display_publish(') - assert uninit.index('apple_t6030_display_withdraw(') < uninit.index('apple_drm_quiesce_connectors(') < uninit.index('drm_dev_unplug(') + assert init.index('ret = drm_dev_register(') < init.index('drm_client_setup_with_fourcc(') < init.index('ret = apple_g15_display_publish(') + assert uninit.index('apple_g15_display_withdraw(') < uninit.index('apple_drm_quiesce_connectors(') < uninit.index('drm_dev_unplug(') assert 'drm_dev_unregister(&apple->drm);\n\t\tgoto err_unbind;' in init - code = '\n'.join([extract.function(drv, 'static int apple_t6030_display_publish('), - extract.function(drv, 'static void apple_t6030_display_withdraw('), + code = '\n'.join([extract.function(drv, 'static struct device **apple_g15_display_slot('), + extract.function(drv, 'static int apple_g15_display_publish('), + extract.function(drv, 'static void apple_g15_display_withdraw('), + extract.function(drv, 'static int apple_g15_link_device('), extract.function(drv, 'int apple_drm_t6030_link_device('), extract.function(glue, 'int asahi_g15_pmp_prepare(struct device *dev)\n')]) variants = [('control', code)] if args.mutations: for name, old, new in [ - ('no-ready-gate', 'if (!apple_t6030_display)', 'if (false)'), - ('no-display-get', 'apple_t6030_display = get_device(dev);', 'apple_t6030_display = dev;'), + ('no-ready-gate', 'if (!*display)', 'if (false)'), + ('no-display-get', '*slot = get_device(dev);', '*slot = dev;'), ('no-display-put', 'put_device(old);', '(void)old;'), - ('replace-live-provider', 'return apple_t6030_display == dev ? 0 : -EBUSY;', 'apple_t6030_display = dev;'), + ('replace-live-provider', 'return *slot == dev ? 0 : -EBUSY;', '*slot = dev;'), ('ignore-display-error', 'if (ret)\n\t\treturn ret;', 'if (false)\n\t\treturn ret;'), - ('wrong-gpu', '!of_device_is_compatible(consumer->of_node, "apple,agx-t6030")', 'false'), - ('withdraw-wrong-provider', 'if (apple_t6030_display == dev)', 'if (((void)dev, true))'), + ('wrong-gpu', '!of_device_is_compatible(consumer->of_node, gpu)', '((void)gpu, false)'), + ('withdraw-wrong-provider', 'if (*slot == dev)', 'if (((void)dev, true))'), + ('shared-display-slot', 'return &apple_t6031_display;', 'return &apple_t6030_display;'), ]: assert code.count(old) == 1, (name, code.count(old)) variants.append((name, code.replace(old, new))) From dfc916868c9bb4912d7213a75b4b05c247c7d196 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:32:42 -0700 Subject: [PATCH 25/46] soc: apple: t6030-display-gate: default the T6031 native display on J516C The T6031 native display gate ran only with apple_t6031_display.enable=1 on the kernel command line, so a J516C whose boot loader hands the display over still stayed on the boot framebuffer, and with it the G15 GPU, which waits for that display before its first firmware start. Make the T6031 switch tri-state: unset, the native display is requested on the J516C (root compatible apple,j516c) and not on the other T6031 boards. An explicit apple_t6031_display.enable=0 or =1 wins on every T6031 board, and a value kstrtobool() rejects preserves the previous explicit choice, or the board default when unset. The request is resolved once in gate_run(), before the existing checks. The handoff marker, the boot loader's PMP values, the PMP image, the power layout and every refusal after them are unchanged, so a J516C without a complete handoff keeps the framebuffer and the GPU keeps waiting. T6030, T8122, J514C and T6034 behaviour is unchanged. Update the help, the device tree comment and the G15 wait message, which said that =1 is always needed. The KUnit suite saves and restores the switch and the default board, checks the switch values and their resolution, and runs the default-board path through gate_run() with the test tree's own root: on by default, off with =0, and untouched without the handoff marker. Generated-by: LLM Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 41aa66da56ba28868732848bc88544bf0c445df4) [ryan: applied to our drivers/soc/apple/display-gate.c, display-gate-test.c and APPLE_DISPLAY_GATE, the renamed T6030 gate] Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t6031-dcp.dtsi | 7 +- drivers/gpu/drm/asahi/g15/glue.c | 6 +- drivers/soc/apple/Kconfig | 16 +++-- drivers/soc/apple/display-gate-test.c | 86 +++++++++++++++++++++--- drivers/soc/apple/display-gate.c | 49 +++++++++++--- 5 files changed, 132 insertions(+), 32 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6031-dcp.dtsi b/arch/arm64/boot/dts/apple/t6031-dcp.dtsi index 6d3b53cd95e57..f679c57b97709 100644 --- a/arch/arm64/boot/dts/apple/t6031-dcp.dtsi +++ b/arch/arm64/boot/dts/apple/t6031-dcp.dtsi @@ -9,9 +9,10 @@ * interrupts and DART windows are those of the J516C ADT. * * Every node stays status = "disabled". The display gate enables the DCP, - * its mailbox, the two DARTs and the display subsystem only when - * apple_t6031_display.enable=1 is on the kernel command line and the boot - * loader has set apple,t6031-handoff = <1>. Compatibles are t6031-specific, + * its mailbox, the two DARTs and the display subsystem only when the boot + * loader has set apple,t6031-handoff = <1> and the native display is + * requested: by default on the J516C, with apple_t6031_display.enable=1 on + * the other boards, never with apple_t6031_display.enable=0. Compatibles are t6031-specific, * so a kernel that does not know them leaves the nodes unbound. * * Only the T6031 board files include this file: t6034.dtsi includes diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index adda36d4bac26..d77bd8e10f101 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -148,8 +148,8 @@ int asahi_g15_pmp_prepare(struct device *dev) /* * T6031 has no PMP vote, but its first firmware start follows the native - * display as T6030's does: defer until that display is up. Its display is - * opt-in (apple_t6031_display.enable=1), so say once why the GPU waits. + * display as T6030's does: defer until that display is up. The display can + * be off for this boot (no handoff, or not requested), so say once why. */ int asahi_g15_display_prepare(struct device *dev) { @@ -163,7 +163,7 @@ int asahi_g15_display_prepare(struct device *dev) return ret; np = of_find_compatible_node(NULL, NULL, "apple,t6031-display-subsystem"); if (!np || !of_device_is_available(np)) - dev_info_once(dev, "G15: waiting for the T6031 native display, which this boot did not enable (apple_t6031_display.enable=1 and the boot loader's display handoff)\n"); + dev_info_once(dev, "G15: waiting for the T6031 native display, which this boot did not enable (it needs the boot loader's display handoff, and apple_t6031_display.enable=1 except on the J516C)\n"); of_node_put(np); return ret; } diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index ba8b498786414..76641d17a506d 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -274,9 +274,10 @@ config APPLE_DISPLAY_GATE nomodeset. It also serves T6031 (M3 Max) device trees with apple,t6031-handoff, - in the same way, but only for a boot with - apple_t6031_display.enable=1 on the kernel command line, and only - with the PMP values the boot loader passes. + in the same way and only with the PMP values the boot loader passes. + There the native display is on by default on the J516C, unless + apple_t6031_display.enable=0; the other T6031 boards need + apple_t6031_display.enable=1 on the kernel command line. On T6041 (M4 Pro) device trees it enables the internal display nodes only when the boot loader has handed the running display coprocessor @@ -303,10 +304,11 @@ config APPLE_DISPLAY_GATE_KUNIT_TEST default KUNIT_ALL_TESTS help Runs the display gate on test device trees with the T6030, T6031 - and T8122 display nodes, without Apple hardware. On T6031: off by - default, enabled only by apple_t6031_display.enable=1, only with the - boot loader's handoff marker and this Mac's PMP values, and with - nothing changed when a check fails. On T6030: on by default, as + and T8122 display nodes, without Apple hardware. On T6031: on by + default only on the default board (the J516C), otherwise only with + apple_t6031_display.enable=1, off with apple_t6031_display.enable=0, + only with the boot loader's handoff marker and this Mac's PMP values, + and with nothing changed when a check fails. On T6030: on by default, as before, whatever the T6031 option says. On T8122: on by default, and nothing changed when the handoff or the device tree's own PMP topology does not check out. diff --git a/drivers/soc/apple/display-gate-test.c b/drivers/soc/apple/display-gate-test.c index 73b36e5a722b6..c88a338fdc2cd 100644 --- a/drivers/soc/apple/display-gate-test.c +++ b/drivers/soc/apple/display-gate-test.c @@ -7,9 +7,11 @@ * - T6030 (t6030-gate-test.dtso): on by default, as before, and * the T6031 switch does not change it. Run first: the T6031 and T8122 * trees describe a PMP, and the T6030 gate refuses to add a second one. - * - T6031 (t6031-gate-test.dtso): off by default, enabled only by - * apple_t6031_display.enable=1 with the full handoff, and nothing changed - * when a check fails. + * - T6031 (t6031-gate-test.dtso): on by default only on the default board + * (the J516C; the cases point it at the test tree's own root to run that + * path), otherwise only with apple_t6031_display.enable=1, never with + * apple_t6031_display.enable=0; always with the full handoff, and nothing + * changed when a check fails. * - T8122 (t8122-gate-test.dtso): on by default, with the full handoff, * and nothing changed when a check of the handoff or of the reused PMP * topology fails. @@ -143,6 +145,8 @@ struct gate_test { const struct gt_soc *s; struct of_changeset cs; /* this case's own edits to the test tree */ bool requested, t6031_requested; + int t6031_switch; + const char *t6031_board; }; static struct device_node *__init gt_node(struct kunit *test, const char *path) @@ -392,6 +396,8 @@ static int __init gt_init(struct kunit *test, const struct gt_soc *s) /* As the boot left them: the test kernel's command line sets neither. */ t->requested = gate_requested; t->t6031_requested = gate_t6031_requested; + t->t6031_switch = gate_t6031_switch; + t->t6031_board = gate_t6031_board; of_changeset_init(&t->cs); gate_soc = s->soc; gate_pmp_values = NULL; @@ -399,6 +405,7 @@ static int __init gt_init(struct kunit *test, const struct gt_soc *s) gate_pmp_reused = false; gate_requested = true; gate_t6031_requested = false; + gate_t6031_switch = -1; if (gt_live_soc) kunit_skip(test, "the booted device tree has its own /soc"); return 0; @@ -444,6 +451,8 @@ static void __init gt_exit(struct kunit *test) gate_pmp_values = NULL; gate_requested = t->requested; gate_t6031_requested = t->t6031_requested; + gate_t6031_switch = t->t6031_switch; + gate_t6031_board = t->t6031_board; gate_soc = NULL; } @@ -455,29 +464,82 @@ static void __init gate_t6031_param(struct kunit *test) struct gate_test *t = test->priv; KUNIT_EXPECT_FALSE(test, t->t6031_requested); + KUNIT_EXPECT_EQ(test, t->t6031_switch, -1); + KUNIT_EXPECT_STREQ(test, t->t6031_board, "apple,j516c"); KUNIT_EXPECT_TRUE(test, t->requested); KUNIT_EXPECT_EQ(test, gate_setup_t6031("1"), 0); - KUNIT_EXPECT_TRUE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 1); KUNIT_EXPECT_EQ(test, gate_setup_t6031("0"), 0); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 0); + /* A rejected value sets nothing: neither an explicit 1 nor a reset to unset. */ KUNIT_EXPECT_NE(test, gate_setup_t6031("maybe"), 0); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 0); KUNIT_EXPECT_NE(test, gate_setup_t6031(NULL), 0); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 0); + gate_t6031_switch = -1; + KUNIT_EXPECT_NE(test, gate_setup_t6031("maybe"), 0); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, -1); /* The T6030 and T8122 escapes do not touch it, nor it them. */ KUNIT_EXPECT_EQ(test, gate_setup("0"), 0); KUNIT_EXPECT_FALSE(test, gate_requested); - KUNIT_EXPECT_FALSE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, -1); KUNIT_EXPECT_EQ(test, gate_setup_t6031("1"), 0); KUNIT_EXPECT_FALSE(test, gate_requested); KUNIT_EXPECT_EQ(test, gate_setup_t8122("1"), 0); KUNIT_EXPECT_TRUE(test, gate_requested); - KUNIT_EXPECT_TRUE(test, gate_t6031_requested); + KUNIT_EXPECT_EQ(test, gate_t6031_switch, 1); +} + +/* An explicit 0 or 1 wins; unset, only the default board asks for it. */ +static void __init gate_t6031_resolution(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, gate_t6031_wanted(-1, true)); + KUNIT_EXPECT_FALSE(test, gate_t6031_wanted(-1, false)); + KUNIT_EXPECT_FALSE(test, gate_t6031_wanted(0, true)); + KUNIT_EXPECT_FALSE(test, gate_t6031_wanted(0, false)); + KUNIT_EXPECT_TRUE(test, gate_t6031_wanted(1, true)); + KUNIT_EXPECT_TRUE(test, gate_t6031_wanted(1, false)); +} + +/* Points the default board at the test tree's root, so gate_run() takes that path. */ +static void __init gt_default_board(struct kunit *test) +{ + const char *root = NULL; + + if (of_property_read_string(of_root, "compatible", &root) || !root) + kunit_skip(test, "the test tree's root has no compatible"); + gate_t6031_board = root; +} + +/* On the default board, the full handoff and no switch: enabled. */ +static void __init gate_t6031_default_board_on(struct kunit *test) +{ + gt_default_board(test); + gt_run(test); + gt_expect_enabled(test); +} + +/* apple_t6031_display.enable=0 still wins on the default board. */ +static void __init gate_t6031_default_board_switched_off(struct kunit *test) +{ + gt_default_board(test); + KUNIT_ASSERT_EQ(test, gate_setup_t6031("0"), 0); + gt_run(test); + gt_expect_untouched(test); +} + +/* The default asks for the display; it does not stand in for the handoff. */ +static void __init gate_t6031_default_board_no_marker(struct kunit *test) +{ + gt_default_board(test); + gt_remove(test, gt_t6031.dcp, "apple,t6031-handoff"); + gt_run(test); + gt_expect_untouched(test); } -/* With a complete handoff and no command line switch, nothing changes. */ +/* Off the default board, with a complete handoff and no switch, nothing changes. */ static void __init gate_t6031_off_by_default(struct kunit *test) { gt_run(test); @@ -826,7 +888,11 @@ static void __init gate_t8122_enabled_again(struct kunit *test) /* The case and suite structs outlive init: only the functions are init code. */ static struct kunit_case gate_t6031_cases[] __refdata = { KUNIT_CASE(gate_t6031_param), + KUNIT_CASE(gate_t6031_resolution), KUNIT_CASE(gate_t6031_off_by_default), + KUNIT_CASE(gate_t6031_default_board_on), + KUNIT_CASE(gate_t6031_default_board_switched_off), + KUNIT_CASE(gate_t6031_default_board_no_marker), KUNIT_CASE(gate_t6031_switched_off), KUNIT_CASE(gate_t6031_enabled), KUNIT_CASE(gate_t6031_no_dcp_marker), diff --git a/drivers/soc/apple/display-gate.c b/drivers/soc/apple/display-gate.c index 2beb8ef39fd1d..fc5ff53fef8b9 100644 --- a/drivers/soc/apple/display-gate.c +++ b/drivers/soc/apple/display-gate.c @@ -53,9 +53,10 @@ * option. * * On T6031 (M3 Max) the gate serves the same handoff, marked - * apple,t6031-handoff, but only when apple_t6031_display.enable=1 is on the - * kernel command line: the native display is off by default there until it - * has been seen to work. The PMP values must come from the boot loader, in + * apple,t6031-handoff. Its default is scoped to the J516C: there the native + * display is on unless apple_t6031_display.enable=0; the other T6031 boards + * need apple_t6031_display.enable=1. The default changes no check of the + * handoff or the PMP below. The PMP values must come from the boot loader, in * /chosen/asahi,t6031-pmp; there is no fallback. Its PMP topology is * reused from the device tree, as on T8122. The T6031 display power * states nest the other way round (the DCP's DISP_FE under DISP_CPU under @@ -160,9 +161,21 @@ static const char gate_t6031_pmp_uuid[] __initconst = "07FF4C5B-CB2E-3DE6-95FF-3 */ static bool gate_requested __initdata = true; -/* apple_t6031_display.enable: the T6031 native display is off unless it is 1. */ +/* + * apple_t6031_display.enable: -1 while it is not on the command line, else 0 + * or 1. Unset, the T6031 native display is on only on gate_t6031_board. + */ +static int gate_t6031_switch __initdata = -1; +static const char *gate_t6031_board __initdata = "apple,j516c"; +/* The resolved request, set by gate_run() on a T6031. */ static bool gate_t6031_requested __initdata; +/* Whether the T6031 native display is requested: an explicit 0 or 1 wins. */ +static bool __init gate_t6031_wanted(int sw, bool default_board) +{ + return sw < 0 ? default_board : sw; +} + /* One SoC's internal display handoff, as the gate checks and completes it */ struct gate_soc { const char *machine; /* root compatible */ @@ -259,8 +272,9 @@ static const struct gate_soc gate_t8122 __initconst = { }; /* - * T6031 (M3 Max): the T6030 handoff at the T6031 addresses, off unless - * apple_t6031_display.enable=1. The PMP values must come from the boot + * T6031 (M3 Max): the T6030 handoff at the T6031 addresses, on by default on + * the J516C and off unless apple_t6031_display.enable=1 on the other boards; + * apple_t6031_display.enable=0 keeps it off everywhere. The PMP values must come from the boot * loader: there is no built-in fallback. The PMP image is T6031's own; * any other, the T6030 image included, is refused. The report driver checks * the image's SRAM layout before it starts the PMP. The internal display's @@ -368,10 +382,18 @@ static int __init gate_setup_t8122(char *arg) } early_param("apple_t8122_display.enable", gate_setup_t8122); -/* Its own switch: the T6030 and T8122 options do not change it, nor it them. */ +/* + * Its own switch: the T6030 and T8122 options do not change it, nor it them. + * A value kstrtobool() rejects sets nothing, so the board default stays. + */ static int __init gate_setup_t6031(char *arg) { - return kstrtobool(arg, &gate_t6031_requested); + bool on; + int ret = kstrtobool(arg, &on); + + if (!ret) + gate_t6031_switch = on; + return ret; } early_param("apple_t6031_display.enable", gate_setup_t6031); @@ -1993,8 +2015,17 @@ static void __init gate_run(void) if (gate_soc->describe) gate_describe(); + if (gate_soc == &gate_t6031) { + bool board = of_machine_is_compatible(gate_t6031_board); + + gate_t6031_requested = gate_t6031_wanted(gate_t6031_switch, board); + if (gate_t6031_switch < 0 && board) + gate_info("native display on by default on %s; apple_t6031_display.enable=0 keeps the boot framebuffer\n", + gate_t6031_board); + } + if (!*gate_soc->requested) { - if (gate_soc->opt_in) + if (gate_soc->opt_in && gate_t6031_switch < 0) gate_info("native display not requested (%s is not on the command line), display stays on the boot framebuffer\n", gate_soc->opt_in); else From ca05e409505991064b77501220f86f184cee1a9f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:13:23 +0000 Subject: [PATCH 26/46] drm/asahi: g15: say when the T6031 GPU waits on an enabled display A T6031 GPU starts its firmware only after the native display is published, so apple_t6031_display.enable=0, or a display that never comes up, also keeps the GPU off for that boot. The Kconfig help and the display gate's comments described the switch as display-only. Say that it also keeps the GPU off. The only G15 line while the GPU waited was the one for a display node this boot did not enable. An enabled display that is not published yet, because its DCP refused or failed to bind or it has not registered, left the GPU deferred with nothing from G15. Say once that the GPU is waiting for the display to come up. Return values and the wait itself are unchanged. The display-order host controls now expect that line, once, for an enabled but unpublished display, and a new mutant that drops it is caught. Generated-by: LLM Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit ca6c77a09ff0cd5a0b29bab93c736ada827dc865) [ryan: applied to our drivers/soc/apple/display-gate.c and APPLE_DISPLAY_GATE, the renamed T6030 gate] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/glue.c | 7 ++++++- drivers/soc/apple/Kconfig | 6 +++++- drivers/soc/apple/display-gate.c | 8 ++++++-- tools/asahi/g15-display-order-tests.py | 5 ++++- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index d77bd8e10f101..095b9cb96d51f 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -149,7 +149,10 @@ int asahi_g15_pmp_prepare(struct device *dev) /* * T6031 has no PMP vote, but its first firmware start follows the native * display as T6030's does: defer until that display is up. The display can - * be off for this boot (no handoff, or not requested), so say once why. + * be off for this boot (no handoff, or not requested), so say once why. An + * enabled display that is not published yet is said once too: if it never + * comes up (its DCP refused or failed to bind), that line is why the GPU + * stays off. */ int asahi_g15_display_prepare(struct device *dev) { @@ -164,6 +167,8 @@ int asahi_g15_display_prepare(struct device *dev) np = of_find_compatible_node(NULL, NULL, "apple,t6031-display-subsystem"); if (!np || !of_device_is_available(np)) dev_info_once(dev, "G15: waiting for the T6031 native display, which this boot did not enable (it needs the boot loader's display handoff, and apple_t6031_display.enable=1 except on the J516C)\n"); + else + dev_info_once(dev, "G15: waiting for the T6031 native display to come up: the GPU firmware starts only after the display is published\n"); of_node_put(np); return ret; } diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 76641d17a506d..7a99bd1be95a7 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -277,7 +277,11 @@ config APPLE_DISPLAY_GATE in the same way and only with the PMP values the boot loader passes. There the native display is on by default on the J516C, unless apple_t6031_display.enable=0; the other T6031 boards need - apple_t6031_display.enable=1 on the kernel command line. + apple_t6031_display.enable=1 on the kernel command line. The T6031 + GPU driver starts its firmware only after this native display is + up, so apple_t6031_display.enable=0, or a native display that does + not come up, also keeps the GPU off for that boot. It is not a + display-only switch. On T6041 (M4 Pro) device trees it enables the internal display nodes only when the boot loader has handed the running display coprocessor diff --git a/drivers/soc/apple/display-gate.c b/drivers/soc/apple/display-gate.c index fc5ff53fef8b9..419842d17f9be 100644 --- a/drivers/soc/apple/display-gate.c +++ b/drivers/soc/apple/display-gate.c @@ -55,7 +55,10 @@ * On T6031 (M3 Max) the gate serves the same handoff, marked * apple,t6031-handoff. Its default is scoped to the J516C: there the native * display is on unless apple_t6031_display.enable=0; the other T6031 boards - * need apple_t6031_display.enable=1. The default changes no check of the + * need apple_t6031_display.enable=1. The T6031 GPU starts its firmware only + * after this native display is up, so enable=0 (or a display that never + * comes up) keeps the GPU off for that boot too: it is not a display-only + * switch. The default changes no check of the * handoff or the PMP below. The PMP values must come from the boot loader, in * /chosen/asahi,t6031-pmp; there is no fallback. Its PMP topology is * reused from the device tree, as on T8122. The T6031 display power @@ -274,7 +277,8 @@ static const struct gate_soc gate_t8122 __initconst = { /* * T6031 (M3 Max): the T6030 handoff at the T6031 addresses, on by default on * the J516C and off unless apple_t6031_display.enable=1 on the other boards; - * apple_t6031_display.enable=0 keeps it off everywhere. The PMP values must come from the boot + * apple_t6031_display.enable=0 keeps it, and with it the GPU, off everywhere. + * The PMP values must come from the boot * loader: there is no built-in fallback. The PMP image is T6031's own; * any other, the T6030 image included, is refused. The report driver checks * the image's SRAM layout before it starts the PMP. The internal display's diff --git a/tools/asahi/g15-display-order-tests.py b/tools/asahi/g15-display-order-tests.py index 2fd7aa99a2bbe..50e0aa4809396 100644 --- a/tools/asahi/g15-display-order-tests.py +++ b/tools/asahi/g15-display-order-tests.py @@ -165,8 +165,10 @@ def c_source(root): nodes[1] = NULL; CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); } else if (!strcmp(scenario, "pending")) { + /* Enabled but not published: no start, no success, said once. */ disp31.available = true; - CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 0 && links_made == 0); + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); + CHECK(prepare(&g31) == -EPROBE_DEFER && infos == 1 && links_made == 0); } else if (!strcmp(scenario, "ready")) { disp31.available = true; CHECK(publish(&d31) == 0 && d31.refs == 1); @@ -216,6 +218,7 @@ def c_source(root): C_MUTANTS = { "unpublished-succeeds": ("if (!*display)\n\t\treturn -EPROBE_DEFER;", "if (!*display)\n\t\treturn 0;"), "disabled-display-skipped": (re.compile(r"\t\tdev_info_once\(dev,[^;]*;\n"), "\t\t{ of_node_put(np); return 0; }\n"), + "pending-display-silent": (re.compile(r"\telse\n\t\tdev_info_once\(dev,[^;]*;\n"), ""), "t6031-reads-t6030-slot": ("&apple_t6031_display);", "&apple_t6030_display);"), "t6031-published-as-t6030": ("return &apple_t6031_display;", "return &apple_t6030_display;"), "no-machine-check": ("!of_machine_is_compatible(machine) ||", ""), From cc35ef24cf7b66742aa9cd9142ff55cf0e66cf2f Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Wed, 28 Oct 2026 06:15:42 +0100 Subject: [PATCH 27/46] drm/asahi: g15: build Board::DeviceTree only with the G15 backend The T8122 runtime builds g15/hw.rs without the G15 backend, which has no board module, so DRM_ASAHI_M3_T8122=y with DRM_ASAHI_G15=n fails: error[E0433]: failed to resolve: could not find `board` in `super` --> drivers/gpu/drm/asahi/g15/hw.rs:54:32 Only the G15 backend builds device-tree board data. Fixes: aeffd0e53b42 ("drm/asahi: g15: t6031: build the J516C board data without a board file") Signed-off-by: Eryk Wieliczko Assisted-by: Claude:claude-opus-5-5 (cherry picked from commit 6c9836868d269605a4cb0e2367a1281691decf0e) [ryan: pin our hw.rs bytes in the zero-words harness] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/hw.rs | 1 + tools/asahi/g15-zero-words-tests.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index 833f638e14c6c..83f80fda71c90 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -50,6 +50,7 @@ pub(crate) enum Board { /// ([`super::board::record`], [`super::board::device_tree`]), with the SoC's reference /// words for what neither carries ([`super::board::record::Reference`]). A value the /// builder refuses stops admission, before any firmware is touched. + #[cfg(CONFIG_DRM_ASAHI_G15)] DeviceTree(&'static super::board::record::Reference), /// No board data: admission refuses the SoC before anything firmware-visible. None, diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index f4f48aa3cafab..7fb4beb6a3848 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -10,7 +10,9 @@ constant PB_BLOCKS removed), and when the T6031 board source became the device tree with reference words instead of a board file (the Board enum; the T6030 tables are unchanged), and for publication-bound slots (removal of -MAX_VMS and its lifetime-slot comment only). Its T6030 configuration module is +MAX_VMS and its lifetime-slot comment only), and when Board::DeviceTree was +gated on CONFIG_DRM_ASAHI_G15 for the T8122 runtime without the G15 backend +(one #[cfg] line; the tables are unchanged). Its T6030 configuration module is independently pinned to the original bytes. prepare_board() is pinned too; it was refreshed for the same Board change (its T6030 arm is unchanged). """ @@ -28,7 +30,7 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "105395216669292954334e2bd0ecd8cd2dcb6c6faf3fc0ab95d7b5a36436b6c2", + "hw.rs": "1e15a6882971d9888677b4862d0e77a461cb9adf3d948fb1ab11c1f20b6df459", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { From 35d8cd7020cd568aef77d5bc1e53badf89766f3d Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Wed, 28 Oct 2026 05:08:08 +0100 Subject: [PATCH 28/46] drm/asahi: build Uat::try_bind only with the G15 backend Only the G15 backend binds VMs at publication. With DRM_ASAHI_M3_T8122=y and DRM_ASAHI_G15=n, W=1 warns that Uat::try_bind is never used. Signed-off-by: Eryk Wieliczko Assisted-by: Claude:claude-opus-5-5 (cherry picked from commit 88e3cc7d7a618f945d3471b2dcd9f33ce9b47773) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/mmu.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index df123e8ec64e8..8a52174bbc6c6 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -3252,6 +3252,7 @@ impl Uat { } /// Binds a VM without waiting for another slot owner to retire. + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn try_bind(&self, vm: &Vm) -> Result { self.bind_inner(vm, false) } From af28934a8bc6d9c861b2a6db32593dfab59a716d Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Wed, 28 Oct 2026 06:15:42 +0100 Subject: [PATCH 29/46] drm/asahi: build VmReadGuard::covers_range only with the G15 backend Only the G15 command checks use it. With DRM_ASAHI_M3_T8122=y and DRM_ASAHI_G15=n, W=1 warns that it is never used. Signed-off-by: Eryk Wieliczko Assisted-by: Claude:claude-opus-5-5 (cherry picked from commit f5d723338c5cd63d56176959b8a67d97d9fa22df) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/mmu.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 8a52174bbc6c6..997b26de6b2c9 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -416,6 +416,7 @@ pub(crate) struct VmReadGuard(VmExecGuard); #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] impl VmReadGuard { + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { Vm::covers_range_locked(&mut self.0, address, size, read, write) } From deb10a56a9c63de17ddebcd1fdec5ac8c689534c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:14:34 -0700 Subject: [PATCH 30/46] tests: asahi: synchronize tbnet refill-worker teardown checks retry-running-teardown assumes its stop thread has entered cancellation within a fixed 30 ms sleep. If the thread is not scheduled in that window, the control aborts before teardown runs. It also reads cancel_waiting without the fixture mutex. Broadcast the fixture condition when cancellation starts and wait for that event under the mutex, with a bounded two-second deadline. Keep the retry worker paused until the teardown assertions have run. A missing cancel still fails rather than timing out the entire suite. Fixes: 71700a159448 ("net: thunderbolt: retry receive refill after allocation failure") Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> Generated-by: LLM (cherry picked from commit a9fa0bb710fea492a7ede4d341a929d510bfd7c1) Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/tbnet-refill-fixture.h | 13 ++++++++++++- tools/asahi/tbnet-refill-tests.py | 14 +++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tools/asahi/tbnet-refill-fixture.h b/tools/asahi/tbnet-refill-fixture.h index e88dfd62831db..d9dee1baab9b2 100644 --- a/tools/asahi/tbnet-refill-fixture.h +++ b/tools/asahi/tbnet-refill-fixture.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #define TBNET_RING_SIZE 256 #define TBNET_FRAME_SIZE 4096 @@ -92,7 +93,17 @@ static bool in_softirq(void){return fake_softirq;} static int smp_processor_id(void){return 0;} static void schedule_work(struct work_struct *w){} static bool queue_delayed_work(int wq,struct delayed_work *w,unsigned long delay){assert(delay==100);pthread_mutex_lock(&gate);bool new=!w->pending;if(new){w->pending=true;retry_queues++;retry_delay=delay;}pthread_mutex_unlock(&gate);return new;} -static void cancel_delayed_work_sync(struct delayed_work *w){pthread_mutex_lock(&gate);cancel_waiting=true;while(w->running)pthread_cond_wait(&cv,&gate);w->pending=false;cancel_waiting=false;pthread_mutex_unlock(&gate);} +static void cancel_delayed_work_sync(struct delayed_work *w) +{ + pthread_mutex_lock(&gate); + cancel_waiting = true; + pthread_cond_broadcast(&cv); + while (w->running) + pthread_cond_wait(&cv, &gate); + w->pending = false; + cancel_waiting = false; + pthread_mutex_unlock(&gate); +} static void fire_retry(struct delayed_work *w){pthread_mutex_lock(&gate);assert(w->pending&&!w->running);w->pending=false;w->running=true;pthread_mutex_unlock(&gate);w->work.fn(&w->work);pthread_mutex_lock(&gate);w->running=false;pthread_cond_broadcast(&cv);pthread_mutex_unlock(&gate);} static void nhi_ring_interrupt_mask(struct tb_ring *r,bool mask){r->masked=mask;} diff --git a/tools/asahi/tbnet-refill-tests.py b/tools/asahi/tbnet-refill-tests.py index f5a6e98e64175..641f8758bc927 100644 --- a/tools/asahi/tbnet-refill-tests.py +++ b/tools/asahi/tbnet-refill-tests.py @@ -56,6 +56,18 @@ def extract(name,text=source): static void *setup_thread(void *p){pthread_mutex_lock(&gate);setup_active=true;pthread_mutex_unlock(&gate);connect(p);pthread_mutex_lock(&gate);setup_active=false;pthread_cond_broadcast(&cv);pthread_mutex_unlock(&gate);return NULL;} static void *tear_thread(void *p){tbnet_tear_down(p,false);return NULL;} static void wait_paused(void){pthread_mutex_lock(&gate);while(!paused)pthread_cond_wait(&cv,&gate);pthread_mutex_unlock(&gate);} +static void wait_cancel(void) +{ + struct timespec deadline; + + assert(!clock_gettime(CLOCK_REALTIME, &deadline)); + deadline.tv_sec += 2; + pthread_mutex_lock(&gate); + while (!cancel_waiting) + assert(!pthread_cond_timedwait(&cv, &gate, &deadline)); + assert(ring_stops == 0 && pool_destroy_calls == 0); + pthread_mutex_unlock(&gate); +} static void release_poll(void){pthread_mutex_lock(&gate);resume_poll=1;pthread_cond_broadcast(&cv);pthread_mutex_unlock(&gate);} int main(int argc,char **argv){assert(argc==2);char *c=argv[1];struct tbnet n;struct net_device d;struct tb tb={0};struct tb_xdomain x={&tb};struct tb_service s={0};struct device dev={&s}; if(!strcmp(c,"pool-create-failure")){pool_error=1;memset(&n,0,sizeof(n));memset(&d,0,sizeof(d));d.priv=&n;n.dev=&d;n.xd=&x;n.svc=&s;pthread_mutex_init(&n.connection_lock,NULL);assert(tbnet_open(&d)==-ENOMEM);assert(!n.rx_ring.pool&&!n.rx_ring.ring&&!n.tx_ring.ring&&ring_frees==2&&enable_calls==0);return 0;} @@ -80,7 +92,7 @@ def extract(name,text=source): if(!strcmp(c,"retry-stop")||!strcmp(c,"retry-disconnect")||!strcmp(c,"retry-suspend")||!strcmp(c,"retry-shutdown")){exhaust(&n);int a=alloc_calls;if(!strcmp(c,"retry-stop"))tbnet_stop(&d);else if(!strcmp(c,"retry-suspend"))tbnet_suspend(&dev);else if(!strcmp(c,"retry-shutdown"))tbnet_shutdown(&s);else tbnet_tear_down(&n,false);assert(!n.rx_refill_work.pending&&!n.napi.enabled&&alloc_calls==a);assert(!n.rx_ring.ring||!n.rx_ring.ring->running);return 0;} if(!strcmp(c,"retry-reconnect")){exhaust(&n);tbnet_tear_down(&n,false);assert(!n.rx_refill_work.pending);alloc_limit=-1;n.login_sent=n.login_received=true;n.remote_transmit_path=12;connect(&n);assert(n.napi.enabled&&n.rx_ring.prod==256&&!n.rx_refill_work.pending);tbnet_stop(&d);return 0;} if(!strcmp(c,"retry-completion-tail")){alloc_limit=0;complete_frames(&n,1,2);pause_at=2;pthread_t a,b;assert(!pthread_create(&a,NULL,poll_thread,&n));wait_paused();assert(n.rx_refill_work.pending);assert(!pthread_create(&b,NULL,stop_thread,&n));usleep(30000);assert(ring_stops==0&&n.rx_refill_work.pending);release_poll();assert(!pthread_join(a,NULL)&&!pthread_join(b,NULL));assert(!n.rx_refill_work.pending&&ring_stops==2);return 0;} - if(!strcmp(c,"retry-running-teardown")){exhaust(&n);pause_at=6;pthread_t a,b;assert(!pthread_create(&a,NULL,retry_thread,&n));wait_paused();assert(!pthread_create(&b,NULL,stop_thread,&n));usleep(30000);assert(ring_stops==0&&pool_destroy_calls==0&&cancel_waiting);release_poll();assert(!pthread_join(a,NULL)&&!pthread_join(b,NULL));assert(!n.rx_refill_work.pending&&!n.rx_refill_work.running&&!n.napi.owner&&ring_stops==2&&pool_destroy_calls==1);return 0;} + if(!strcmp(c,"retry-running-teardown")){exhaust(&n);pause_at=6;pthread_t a,b;assert(!pthread_create(&a,NULL,retry_thread,&n));wait_paused();assert(!pthread_create(&b,NULL,stop_thread,&n));wait_cancel();release_poll();assert(!pthread_join(a,NULL)&&!pthread_join(b,NULL));assert(!n.rx_refill_work.pending&&!n.rx_refill_work.running&&!n.napi.owner&&ring_stops==2&&pool_destroy_calls==1);return 0;} if(!strcmp(c,"retry-already-polling")){exhaust(&n);tbnet_start_poll(&n);fire_retry(&n.rx_refill_work);int rearm=rearm_calls;assert(n.napi.missed&&tbnet_poll(&n.napi,64)==0&&n.napi.owner&&n.rx_refill_work.pending&&rearm_calls==rearm);alloc_limit=-1;assert(tbnet_poll(&n.napi,64)==0&&!n.napi.owner&&n.rx_ring.prod-n.rx_ring.cons==256);return 0;} if(!strcmp(c,"retry-disabled-worker")){exhaust(&n);napi_disable(&n.napi);int a=alloc_calls,sch=schedule_calls;fire_retry(&n.rx_refill_work);assert(!n.napi.owner&&schedule_calls==sch&&alloc_calls==a);return 0;} if(!strcmp(c,"retry-counter-wrap")){n.rx_ring.cons=0xffffff00;n.rx_ring.prod=0;exhaust(&n);alloc_limit=-1;retry_poll(&n);assert(n.rx_ring.cons==0&&n.rx_ring.prod==256&&!n.rx_refill_work.pending);return 0;} From bd0eec990a224a088601e64cdc7d1eb5b73f5f1d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:31:50 -0700 Subject: [PATCH 31/46] tests: asahi: scan default T6030 trees for PMP compatibles The display gate refuses any existing PMP or report compatible anywhere in the tree, including disabled nodes. The DT control only checks four names under /soc, so renamed or nested nodes can pass it and still block the runtime PMP overlay. Walk the whole default-profile tree and reject either compatible in any entry of each compatible string list. Keep the existing static-profile node and alias checks. Compare case-insensitively, as the OF lookup does. Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> Generated-by: LLM (cherry picked from commit 212c155f323fb12b025d1c6ec880f557e9fa077f) [ryan: drop the Link to the omacom review discussion] Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/t6030-pmp-dt-tests.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tools/asahi/t6030-pmp-dt-tests.py b/tools/asahi/t6030-pmp-dt-tests.py index 22826e998173d..913e8d31aa7d7 100644 --- a/tools/asahi/t6030-pmp-dt-tests.py +++ b/tools/asahi/t6030-pmp-dt-tests.py @@ -15,6 +15,32 @@ "pmp@350500000", "mbox@350c08000", } +PMP_COMPATIBLES = {"apple,t6000-pmp-v2", "apple,t6030-pmp-v2-report"} + + +def check_default_compatibles(dtb, board): + pending = ["/"] + while pending: + node = pending.pop() + raw = subprocess.check_output( + ["fdtget", "-t", "bx", "-d", "", str(dtb), node, "compatible"], + text=True, + ) + # Match the gate's whole-tree OF lookup, including disabled nodes + # and every string in a compatible list. + compatible = bytes(int(value, 16) for value in raw.split()) + matches = PMP_COMPATIBLES.intersection( + value.decode("ascii").lower() for value in compatible.split(b"\0") + if value + ) + if matches: + raise AssertionError(f"{board} default: static PMP compatible " + f"{sorted(matches)} at {node}") + children = subprocess.check_output( + ["fdtget", "-l", str(dtb), node], text=True, + ) + pending.extend(node.rstrip("/") + "/" + child + for child in children.splitlines()) def check_profile(tree, output, dtc, board, static): @@ -38,6 +64,8 @@ def check_profile(tree, output, dtc, board, static): if present != expected: raise AssertionError(f"{board} {profile}: static PMP nodes " f"{sorted(present)}, expected {sorted(expected)}") + if not static: + check_default_compatibles(dtb, board) alias = subprocess.run(["fdtget", "-t", "s", str(dtb), "/aliases", "pmp"], capture_output=True, text=True) From 986132e6f58040561e62d1b52dba8ba2b5c811c3 Mon Sep 17 00:00:00 2001 From: Jeppe Bylov Date: Mon, 26 Oct 2026 20:06:47 +0000 Subject: [PATCH 32/46] drm/asahi: g15: pin the B0 firmware on the J514S The 14-inch M3 Pro (J514S) boot chain loads the same G15 B0 firmware image as the J516S: TEXT 0x5c000, DATA 0x114000, GKTS marker 0x58e88 and TEXT sha256 da26eccab1f8598da80c855eca916520406d1d0670f1d470e79bb9b02209db53. The pinned row names only apple,j516s, so on a J514S the build is refused and the GPU only starts with the g15_accept_text_prefix bring-up override. Add the same build as a J514S row. Rows still pin on the machine they name; nothing changes on the J516S or on other machines. Extend the maintained-handoff host test to check both rows. Measured on a J514S (macOS 14.8.3 stub, 7.2.11 on integration/aurora 937b727): the override log line reports the identical size, marker and digest, and the GPU has run with that build since 2026-10-23. Signed-off-by: Jeppe Bylov Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 6523818174f3750334d3f4a403e5b59035931dd8) (cherry picked from commit 9ce2cf328e9a65fbb64b71cb12a2c57d8ee3c733) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/hw.rs | 15 +++++++++++++++ tools/asahi/g15-maintained-handoff-tests.py | 10 ++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index 83f80fda71c90..3768907f7416e 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -160,6 +160,21 @@ pub(crate) mod t6030 { 0xe7, 0x9b, 0xb9, 0xb0, 0x22, 0x09, 0xdb, 0x53, ], }, + // The J514S (14-inch) boot chain loads the same B0 image: its + // measured TEXT digest, sizes and marker match the J516S row above. + FirmwareBuild { + name: "J514S B0 RTKit-2419.140.12 (maintained 14.x handoff)", + machine: Some(kernel::c_str!("apple,j514s")), + text_size: 0x5c000, + data_size: 0x114000, + marker: 0x58e88, + text_sha256: [ + 0xda, 0x26, 0xec, 0xca, 0xb1, 0xf8, 0x59, 0x8d, + 0xa8, 0x0c, 0x85, 0x5e, 0xca, 0x91, 0x65, 0x20, + 0x40, 0x6d, 0x1d, 0x06, 0x70, 0xf1, 0xd4, 0x70, + 0xe7, 0x9b, 0xb9, 0xb0, 0x22, 0x09, 0xdb, 0x53, + ], + }, ], machine: kernel::c_str!("apple,t6030"), platform: super::Platform::AdtWithPmp, diff --git a/tools/asahi/g15-maintained-handoff-tests.py b/tools/asahi/g15-maintained-handoff-tests.py index 0c07c2eda3065..fdafd799645cb 100644 --- a/tools/asahi/g15-maintained-handoff-tests.py +++ b/tools/asahi/g15-maintained-handoff-tests.py @@ -137,10 +137,12 @@ def item(s, marker): OWNER_CALLS.set(0); let bytes = fixture(kind); assert!(parse(&dev, &Adt { data: &bytes }).is_err()); assert_eq!(OWNER_CALLS.get(), 0); } - assert_eq!(BUILDS.len(), 2, "maintained B0 firmware row must not require an absent profile symbol"); - assert_eq!(BUILDS[1].machine.unwrap().to_bytes(), b"apple,j516s"); - assert_eq!((BUILDS[1].text_size, BUILDS[1].data_size, BUILDS[1].marker), (0x5c000, 0x114000, 0x58e88)); - assert_eq!(BUILDS[1].text_sha256, [0xda,0x26,0xec,0xca,0xb1,0xf8,0x59,0x8d,0xa8,0x0c,0x85,0x5e,0xca,0x91,0x65,0x20,0x40,0x6d,0x1d,0x06,0x70,0xf1,0xd4,0x70,0xe7,0x9b,0xb9,0xb0,0x22,0x09,0xdb,0x53]); + assert_eq!(BUILDS.len(), 3, "maintained B0 firmware row must not require an absent profile symbol"); + for (row, machine) in [(1, &b"apple,j516s"[..]), (2, &b"apple,j514s"[..])] { + assert_eq!(BUILDS[row].machine.unwrap().to_bytes(), machine); + assert_eq!((BUILDS[row].text_size, BUILDS[row].data_size, BUILDS[row].marker), (0x5c000, 0x114000, 0x58e88)); + assert_eq!(BUILDS[row].text_sha256, [0xda,0x26,0xec,0xca,0xb1,0xf8,0x59,0x8d,0xa8,0x0c,0x85,0x5e,0xca,0x91,0x65,0x20,0x40,0x6d,0x1d,0x06,0x70,0xf1,0xd4,0x70,0xe7,0x9b,0xb9,0xb0,0x22,0x09,0xdb,0x53]); + } for error in [0, -517, -12] { INIT_ERROR.set(error); let shared = Arc::new(Shared); let result = Rtkit::new(&dev, &shared); assert_eq!(result.is_ok(), error == 0); From 83ca860b6a2643ec0cc0dc1a996527c1cd0cde4d Mon Sep 17 00:00:00 2001 From: Chris Kearney Date: Fri, 30 Oct 2026 04:13:18 +0000 Subject: [PATCH 33/46] test: asahi: refresh the J514S firmware row golden pins The J514S B0 firmware row changes the hw.rs file and its independently pinned T6030 module. Derive both hashes from the current integration tree plus exactly the reviewed 15-line firmware-table insertion. Keep the extraction, assertions and mutation controls unchanged. The maintained-handoff and zero-word mutation suites pass with the new row. Removing the row, changing the J516S digest, or editing outside the module still fails the source guards. Signed-off-by: Chris Kearney (cherry picked from commit d777cc26a244b158a772171d951910b9a9776b01) [ryan: pin our hw.rs bytes; the T6030 module pin is omacom's, as our module carried the same original bytes] Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/g15-zero-words-tests.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index 7fb4beb6a3848..886b0ab3be2c6 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -13,7 +13,10 @@ MAX_VMS and its lifetime-slot comment only), and when Board::DeviceTree was gated on CONFIG_DRM_ASAHI_G15 for the T8122 runtime without the G15 backend (one #[cfg] line; the tables are unchanged). Its T6030 configuration module is -independently pinned to the original bytes. prepare_board() is pinned too; +independently pinned to the original bytes, +except for the J514S B0 FirmwareBuild row added to its firmware table: the +module is the original pinned bytes (0079ae08) plus that one 15-line row, and +the whole hw.rs pin was refreshed with it. prepare_board() is pinned too; it was refreshed for the same Board change (its T6030 arm is unchanged). """ @@ -30,11 +33,11 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "1e15a6882971d9888677b4862d0e77a461cb9adf3d948fb1ab11c1f20b6df459", + "hw.rs": "9bd94ec8654736e1295061925f3c47264be8d338ad4dcb38b8d66b3dfcef2fa7", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { - ("hw.rs", "pub(crate) mod t6030 {"): "0079ae08fbe0e51cbf14e48711981a81048d1a1d3fc5495ab8d5c376283a760c", + ("hw.rs", "pub(crate) mod t6030 {"): "cab592b5d776be9ede93b96b1b0526c1d838ef2948a5e4a86caa4ec71f71c2ca", ("initdata.rs", "fn templates("): "c3115c9926d3d7d93317cb8510e93b04f30b0c256811d7137568e822dd93b8ee", ("initdata.rs", "pub(crate) fn prepare_board("): "ebef64ab87ef7b1f994eb7fa2a2424d05bc4fc9e88cbfbf18e51570bdfc4222d", ("board.rs", "pub(crate) fn from_adt("): "cecd0b28db5bd94d3b75c1bbb5f35204e8bf0699f74e5b3d08228c59025f6ac2", From 7beb9f27478fd4f94ed148c6d52628244f6c4cd7 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:18:00 -0500 Subject: [PATCH 34/46] drm/asahi: expose raw firmware statistics through debugfs The stats channel keeps the firmware words verbatim in a new raw snapshot: every message kind keeps its last sample, a sample counter and the monotonic capture time. No unit is assumed, nothing is scaled and no metric is derived. The snapshot is exported read-only as `asahi_stats` in the DRM device debugfs directory, format tag `asahi_stats_v1`. The interpreted `agx_stats` sysfs export is unchanged, and the raw recording is not gated by the stats_export parameter. The G17 and M3 backends have no stats channel decoder, so they expose no raw file, like their sysfs export. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> (cherry picked from commit ecc22118cb34929db1f13acbdc68c2ea471eafff) [ryan: our driver has no T8122 probe; the debugfs entry is removed by drm_dev_unregister() before the registration data that owns the snapshot is dropped, so the driver data keeps no extra reference; read stats_export by value as on Linux 7.3] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/asahi_core.rs | 1 + drivers/gpu/drm/asahi/channel.rs | 90 +++- drivers/gpu/drm/asahi/driver.rs | 8 + drivers/gpu/drm/asahi/gpu.rs | 23 + drivers/gpu/drm/asahi/stats_raw.rs | 495 ++++++++++++++++++ .../asahi/tests/test_asahi_stats_export.py | 154 ++++++ rust/bindings/bindings_helper.h | 1 + 7 files changed, 770 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/asahi/stats_raw.rs create mode 100755 drivers/gpu/drm/asahi/tests/test_asahi_stats_export.py diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 6afd51ce4fa0a..1e0b043b8b183 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -36,6 +36,7 @@ mod queue; mod regs; mod slotalloc; mod stats; +mod stats_raw; mod sysfs_exports; mod timestamp; mod usage; diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs index 249b10bb361ca..5a14982dca053 100644 --- a/drivers/gpu/drm/asahi/channel.rs +++ b/drivers/gpu/drm/asahi/channel.rs @@ -24,13 +24,17 @@ use crate::{ event, gpu, mem, - module_parameters, // + module_parameters, + stats_raw::RawStatsSnapshot, // }; use core::sync::atomic::Ordering; use kernel::{ c_str, prelude::*, - sync::Arc, + sync::{ + Arc, + SpinLock, // + }, time::{ delay::fsleep, Delta, @@ -588,6 +592,9 @@ pub(crate) struct StatsChannel { dev: AsahiDevRef, ch: RxChannel, snap: Arc, + /// Last raw sample of every statistics kind, shared with the export + /// through `crate::stats_raw`. + raw: Arc>, /// Rate of firmware stats-message timestamps (Hz). All stats timestamps /// run on the chip base clock (`HwConfig::base_clock_hz`, 24 MHz on every /// supported SoC), not nanoseconds. @@ -604,12 +611,14 @@ impl StatsChannel::ver { dev: AsahiDevice, alloc: &mut gpu::KernelAllocators, snap: Arc, + raw: Arc>, ts_hz: u64, ) -> Result { Ok(StatsChannel::ver { dev: dev.clone(), ch: RxChannel::::new(alloc, 0x100)?, snap, + raw, ts_hz, last_util_ts: 0, }) @@ -631,6 +640,7 @@ impl StatsChannel::ver { // accessing the enum view is valid. let msg = unsafe { msg.msg }; cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); + self.record_raw(&msg); if module_parameters::stats_export.value() != 0 { // Retain the latest values for /sys/class/drm/card*/ // device/agx_stats. Read-only with respect to @@ -713,4 +723,80 @@ impl StatsChannel::ver { } } } + + /// Record one statistics message in the raw per-device snapshot. + /// + /// The firmware words are stored exactly as they arrived: no unit is + /// assumed, nothing is scaled and no metric is derived from them. + fn record_raw(&mut self, msg: &StatsMsg::ver) { + let stamp = crate::stats_raw::monotonic_ns(); + let mut snap = self.raw.lock(); + snap.note_message(); + match msg { + StatsMsg::ver::Power { power, .. } => snap.set_power(power.0, stamp), + StatsMsg::ver::PowerOn { off_time, .. } => snap.set_power_on(off_time.0, stamp), + StatsMsg::ver::PowerOff { on_time, .. } => snap.set_power_off(on_time.0, stamp), + StatsMsg::ver::Utilization { + timestamp, + util1, + util2, + util3, + util4, + } => snap.set_utilization(timestamp.0, [*util1, *util2, *util3, *util4], stamp), + StatsMsg::ver::AvgPower { + active_cs, + unk2, + unk3, + unk4, + avg_power, + } => snap.set_avg_power(active_cs.0, [*unk2, *unk3, *unk4], *avg_power, stamp), + StatsMsg::ver::Temperature { + raw_value, + scale, + tmin, + tmax, + .. + } => snap.set_temperature(*raw_value, *scale, *tmin, *tmax, stamp), + StatsMsg::ver::PowerState { + timestamp, + last_busy_ts, + active, + poweroff, + unk1, + pstate, + unk2, + unk3, + } => snap.set_power_state( + timestamp.0, + last_busy_ts.0, + *active, + *poweroff, + *unk1, + *pstate, + *unk2, + *unk3, + stamp, + ), + StatsMsg::ver::FwBusy { timestamp, busy } => { + snap.set_fw_busy(timestamp.0, *busy, stamp) + } + StatsMsg::ver::PState { + ps_min, + unk1, + ps_max, + unk2, + .. + } => snap.set_pstate(*ps_min, *unk1, *ps_max, *unk2, stamp), + StatsMsg::ver::TempSensor { + sensor_id, + raw_value, + scale, + tmin, + tmax, + .. + } => snap.set_temp_sensor(*sensor_id, *raw_value, *scale, *tmin, *tmax, stamp), + // The remaining tags are not decoded by the driver. + _ => {} + } + } } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index a89728cc2f2b3..d6e461a7c3210 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -302,12 +302,20 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); + let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + // The DRM core created this card's debugfs directory when the device was registered; + // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and + // waits for its readers, before the registration data that owns the snapshot is dropped. + if let Some(snapshot) = debug_stats.as_ref() { + crate::stats_raw::register(reg.device(), snapshot); + } + // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index e9f1008fc5a72..a91f267f9e1e0 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -35,6 +35,7 @@ use kernel::{ }, macros::versions, new_mutex, + new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -44,6 +45,7 @@ use kernel::{ }, Arc, Mutex, + SpinLock, UniqueArc, // }, time::{ @@ -84,6 +86,7 @@ use crate::{ queue, regs, stats, + stats_raw, workqueue, // }; @@ -258,6 +261,9 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, + /// Last raw firmware statistics of this device, exported read-only + /// through `stats_raw`. + raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, #[pin] @@ -274,6 +280,11 @@ pub(crate) trait Gpu: Send + Sync { /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; + /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. + fn raw_stats_snapshot(&self) -> Option>> { + None + } + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -827,6 +838,12 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); + // Per-device raw statistics snapshot, shared with the debugfs export. + let raw_stats_snap = Arc::pin_init( + new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), + GFP_KERNEL, + )?; + let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -842,6 +859,7 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, + raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -869,6 +887,7 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), + raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1324,6 +1343,10 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } + fn raw_stats_snapshot(&self) -> Option>> { + Some(self.raw_stats.clone()) + } + fn as_any(&self) -> &dyn Any { self } diff --git a/drivers/gpu/drm/asahi/stats_raw.rs b/drivers/gpu/drm/asahi/stats_raw.rs new file mode 100644 index 0000000000000..8d5de5bf59607 --- /dev/null +++ b/drivers/gpu/drm/asahi/stats_raw.rs @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Raw firmware statistics snapshot and its read-only debugfs export. +//! +//! The AGX firmware sends statistics messages on the stats channel +//! (`channel::StatsChannel`). They are recorded here exactly as they arrive: +//! every field keeps the value the firmware sent, no unit is assumed, nothing +//! is scaled and no metric is derived from it. `FwBusy` is the firmware's own +//! busy flag and is *not* a GPU utilisation figure; the `Utilization` counters +//! are raw per-subqueue counters whose unit this driver does not assert. +//! +//! This complements the interpreted `stats` snapshot behind `agx_stats`: the +//! snapshot here keeps every message kind the firmware sends, together with +//! per-kind sample counters and capture times, and it never derives a value +//! from the firmware words. +//! +//! Every message kind keeps its own sample counter and the monotonic time at +//! which the sample was captured, so a reader can tell a missing sample +//! (count 0) from a stale one (old capture time), and can compute rates and +//! deltas between two reads. +//! +//! The snapshot is fixed size: recording a sample never allocates. It is +//! guarded by a spin lock because the critical section is short: a sample +//! is stored under the lock and a reader copies the snapshot out, so the lock +//! is never held across formatting. + +use core::fmt::Write; + +use kernel::{ + bindings, + prelude::*, + seq_file::SeqFile, + sync::{ + Arc, + SpinLock, // + }, +}; + +use crate::driver::AsahiDrmDevice; + +/// Format tag of the export. Bumped whenever the field set changes. +pub(crate) const STATS_FORMAT: &str = "asahi_stats_v1"; + +/// The firmware statistics message kinds that are recorded. +/// +/// The discriminants index the per-kind counters of [`RawStatsSnapshot`]; the +/// firmware message tag of each kind is given in its documentation, so every +/// exported line can be mapped back to the message it came from. +#[derive(Copy, Clone, PartialEq, Eq)] +#[repr(usize)] +pub(crate) enum StatsKind { + /// 0x00 `Power`. + Power = 0, + /// 0x02 `PowerOn`. + PowerOn = 1, + /// 0x03 `PowerOff`. + PowerOff = 2, + /// 0x04 `Utilization`. + Utilization = 3, + /// 0x09 `AvgPower`. + AvgPower = 4, + /// 0x0a `Temperature`. + Temperature = 5, + /// 0x0b `PowerState`. + PowerState = 6, + /// 0x0c `FwBusy`. + FwBusy = 7, + /// 0x0d `PState`. + PState = 8, + /// 0x0e `TempSensor`. + TempSensor = 9, +} + +impl StatsKind { + /// Number of recorded kinds. + pub(crate) const COUNT: usize = 10; + + /// All kinds, in export order. + const ALL: [StatsKind; StatsKind::COUNT] = [ + StatsKind::Power, + StatsKind::PowerOn, + StatsKind::PowerOff, + StatsKind::Utilization, + StatsKind::AvgPower, + StatsKind::Temperature, + StatsKind::PowerState, + StatsKind::FwBusy, + StatsKind::PState, + StatsKind::TempSensor, + ]; + + /// Stable name of the kind in the export. + fn name(self) -> &'static str { + match self { + StatsKind::Power => "power", + StatsKind::PowerOn => "power_on", + StatsKind::PowerOff => "power_off", + StatsKind::Utilization => "utilization", + StatsKind::AvgPower => "avg_power", + StatsKind::Temperature => "temperature", + StatsKind::PowerState => "power_state", + StatsKind::FwBusy => "fw_busy", + StatsKind::PState => "pstate", + StatsKind::TempSensor => "temp_sensor", + } + } +} + +/// Last received firmware statistics of one device. +/// +/// All values are the raw firmware words. The driver knows the unit of none +/// of them and derives nothing from them. +#[derive(Copy, Clone, Default)] +pub(crate) struct RawStatsSnapshot { + /// Number of samples received per kind (0 means: never received). + count: [u64; StatsKind::COUNT], + /// Monotonic capture time (ns) of the last sample per kind. + captured_ns: [u64; StatsKind::COUNT], + /// Number of statistics messages received on the channel. + messages: u64, + + /// 0x00 `Power`: raw power word, unit unknown. + power: u64, + /// 0x02 `PowerOn`: raw off-time word, unit unknown. + power_off_time: u64, + /// 0x03 `PowerOff`: raw on-time word, unit unknown. + power_on_time: u64, + /// 0x04 `Utilization`: raw firmware timestamp and four raw counters. + util_timestamp: u64, + /// 0x04 `Utilization`: the four raw counters. + util: [u32; 4], + /// 0x09 `AvgPower`: raw active time word, unit unknown. + avg_power_active_cs: u64, + /// 0x09 `AvgPower`: raw undecoded word. + avg_power_unk2: u32, + /// 0x09 `AvgPower`: raw undecoded word. + avg_power_unk3: u32, + /// 0x09 `AvgPower`: raw undecoded word. + avg_power_unk4: u32, + /// 0x09 `AvgPower`: raw average power word, unit unknown. + avg_power: u32, + /// 0x0a `Temperature`: raw sensor word. + temp_raw_value: u32, + /// 0x0a `Temperature`: raw scale word, travels with the sample. + temp_scale: u32, + /// 0x0a `Temperature`: raw minimum word. + temp_tmin: u32, + /// 0x0a `Temperature`: raw maximum word. + temp_tmax: u32, + /// 0x0b `PowerState`: raw firmware timestamp. + power_state_timestamp: u64, + /// 0x0b `PowerState`: raw firmware timestamp of the last busy period. + power_state_last_busy_ts: u64, + /// 0x0b `PowerState`: raw active word. + power_state_active: u32, + /// 0x0b `PowerState`: raw poweroff word. + power_state_poweroff: u32, + /// 0x0b `PowerState`: raw undecoded word. + power_state_unk1: u32, + /// 0x0b `PowerState`: raw p-state index. + power_state_pstate: u32, + /// 0x0b `PowerState`: raw undecoded word. + power_state_unk2: u32, + /// 0x0b `PowerState`: raw undecoded word. + power_state_unk3: u32, + /// 0x0c `FwBusy`: raw firmware timestamp. + fw_busy_timestamp: u64, + /// 0x0c `FwBusy`: raw firmware busy flag. + fw_busy: u32, + /// 0x0d `PState`: raw lowest p-state bound. + pstate_min: u32, + /// 0x0d `PState`: raw undecoded word. + pstate_unk1: u32, + /// 0x0d `PState`: raw highest p-state bound. + pstate_max: u32, + /// 0x0d `PState`: raw undecoded word. + pstate_unk2: u32, + /// 0x0e `TempSensor`: raw sensor identifier. + sensor_id: u32, + /// 0x0e `TempSensor`: raw sensor word. + sensor_raw_value: u32, + /// 0x0e `TempSensor`: raw scale word, travels with the sample. + sensor_scale: u32, + /// 0x0e `TempSensor`: raw minimum word. + sensor_tmin: u32, + /// 0x0e `TempSensor`: raw maximum word. + sensor_tmax: u32, +} + +impl RawStatsSnapshot { + /// Note one statistics message on the channel. + pub(crate) fn note_message(&mut self) { + self.messages += 1; + } + + /// Record one sample of `kind` captured at `stamp` (monotonic ns). + fn record(&mut self, kind: StatsKind, stamp: u64) { + self.count[kind as usize] += 1; + self.captured_ns[kind as usize] = stamp; + } + + /// Record the 0x00 `Power` sample. + pub(crate) fn set_power(&mut self, power: u64, stamp: u64) { + self.power = power; + self.record(StatsKind::Power, stamp); + } + + /// Record the 0x02 `PowerOn` sample. + pub(crate) fn set_power_on(&mut self, off_time: u64, stamp: u64) { + self.power_off_time = off_time; + self.record(StatsKind::PowerOn, stamp); + } + + /// Record the 0x03 `PowerOff` sample. + pub(crate) fn set_power_off(&mut self, on_time: u64, stamp: u64) { + self.power_on_time = on_time; + self.record(StatsKind::PowerOff, stamp); + } + + /// Record the 0x04 `Utilization` sample. + pub(crate) fn set_utilization(&mut self, timestamp: u64, util: [u32; 4], stamp: u64) { + self.util_timestamp = timestamp; + self.util = util; + self.record(StatsKind::Utilization, stamp); + } + + /// Record the 0x09 `AvgPower` sample. + pub(crate) fn set_avg_power( + &mut self, + active_cs: u64, + unk: [u32; 3], + avg_power: u32, + stamp: u64, + ) { + self.avg_power_active_cs = active_cs; + self.avg_power_unk2 = unk[0]; + self.avg_power_unk3 = unk[1]; + self.avg_power_unk4 = unk[2]; + self.avg_power = avg_power; + self.record(StatsKind::AvgPower, stamp); + } + + /// Record the 0x0a `Temperature` sample. + pub(crate) fn set_temperature( + &mut self, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + stamp: u64, + ) { + self.temp_raw_value = raw_value; + self.temp_scale = scale; + self.temp_tmin = tmin; + self.temp_tmax = tmax; + self.record(StatsKind::Temperature, stamp); + } + + /// Record the 0x0b `PowerState` sample. + #[allow(clippy::too_many_arguments)] + pub(crate) fn set_power_state( + &mut self, + timestamp: u64, + last_busy_ts: u64, + active: u32, + poweroff: u32, + unk1: u32, + pstate: u32, + unk2: u32, + unk3: u32, + stamp: u64, + ) { + self.power_state_timestamp = timestamp; + self.power_state_last_busy_ts = last_busy_ts; + self.power_state_active = active; + self.power_state_poweroff = poweroff; + self.power_state_unk1 = unk1; + self.power_state_pstate = pstate; + self.power_state_unk2 = unk2; + self.power_state_unk3 = unk3; + self.record(StatsKind::PowerState, stamp); + } + + /// Record the 0x0c `FwBusy` sample. + pub(crate) fn set_fw_busy(&mut self, timestamp: u64, busy: u32, stamp: u64) { + self.fw_busy_timestamp = timestamp; + self.fw_busy = busy; + self.record(StatsKind::FwBusy, stamp); + } + + /// Record the 0x0d `PState` sample. + pub(crate) fn set_pstate( + &mut self, + ps_min: u32, + unk1: u32, + ps_max: u32, + unk2: u32, + stamp: u64, + ) { + self.pstate_min = ps_min; + self.pstate_unk1 = unk1; + self.pstate_max = ps_max; + self.pstate_unk2 = unk2; + self.record(StatsKind::PState, stamp); + } + + /// Record the 0x0e `TempSensor` sample. + pub(crate) fn set_temp_sensor( + &mut self, + sensor_id: u32, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + stamp: u64, + ) { + self.sensor_id = sensor_id; + self.sensor_raw_value = raw_value; + self.sensor_scale = scale; + self.sensor_tmin = tmin; + self.sensor_tmax = tmax; + self.record(StatsKind::TempSensor, stamp); + } + + /// Write the raw payload fields of one kind. + fn write_payload(&self, kind: StatsKind, out: &mut impl Write) -> core::fmt::Result { + match kind { + StatsKind::Power => write!(out, " power={}", self.power), + StatsKind::PowerOn => write!(out, " off_time={}", self.power_off_time), + StatsKind::PowerOff => write!(out, " on_time={}", self.power_on_time), + StatsKind::Utilization => write!( + out, + " timestamp={} util1={} util2={} util3={} util4={}", + self.util_timestamp, self.util[0], self.util[1], self.util[2], self.util[3] + ), + StatsKind::AvgPower => write!( + out, + " active_cs={} unk2={} unk3={} unk4={} avg_power={}", + self.avg_power_active_cs, + self.avg_power_unk2, + self.avg_power_unk3, + self.avg_power_unk4, + self.avg_power + ), + StatsKind::Temperature => write!( + out, + " raw_value={} scale={} tmin={} tmax={}", + self.temp_raw_value, self.temp_scale, self.temp_tmin, self.temp_tmax + ), + StatsKind::PowerState => write!( + out, + " timestamp={} last_busy_ts={} active={} poweroff={} unk1={} pstate={} unk2={} unk3={}", + self.power_state_timestamp, + self.power_state_last_busy_ts, + self.power_state_active, + self.power_state_poweroff, + self.power_state_unk1, + self.power_state_pstate, + self.power_state_unk2, + self.power_state_unk3 + ), + StatsKind::FwBusy => { + write!(out, " timestamp={} busy={}", self.fw_busy_timestamp, self.fw_busy) + } + StatsKind::PState => write!( + out, + " ps_min={} unk1={} ps_max={} unk2={}", + self.pstate_min, self.pstate_unk1, self.pstate_max, self.pstate_unk2 + ), + StatsKind::TempSensor => write!( + out, + " sensor_id={} raw_value={} scale={} tmin={} tmax={}", + self.sensor_id, self.sensor_raw_value, self.sensor_scale, self.sensor_tmin, self.sensor_tmax + ), + } + } +} + +/// Write the snapshot in the documented text format. +/// +/// One line per record, `key=value` pairs, LF terminated. A kind that was +/// never received is printed with `count=0` and without payload, so a missing +/// sample is never confused with a zero value. `now_ns` and `captured_ns` are +/// monotonic (CLOCK_MONOTONIC) nanoseconds taken by the driver; every other +/// value is a raw firmware word whose unit this driver does not assert. +pub(crate) fn format_snapshot( + snap: &RawStatsSnapshot, + now_ns: u64, + device: &str, + out: &mut impl Write, +) -> core::fmt::Result { + writeln!(out, "{}", STATS_FORMAT)?; + writeln!(out, "device={}", device)?; + writeln!(out, "now_ns={}", now_ns)?; + writeln!(out, "messages count={}", snap.messages)?; + for kind in StatsKind::ALL { + let count = snap.count[kind as usize]; + write!(out, "{} count={}", kind.name(), count)?; + if count != 0 { + write!(out, " captured_ns={}", snap.captured_ns[kind as usize])?; + snap.write_payload(kind, out)?; + } + writeln!(out)?; + } + Ok(()) +} + +/// Current monotonic time in nanoseconds. +pub(crate) fn monotonic_ns() -> u64 { + // SAFETY: `ktime_get()` may be called from any context. + unsafe { bindings::ktime_get() as u64 } +} + +/// Publish the firmware statistics of one device as a read-only debugfs file. +/// +/// The file is created in the DRM device's own debugfs directory +/// (`/sys/kernel/debug/dri//asahi_stats`), so a card only ever exposes +/// its own statistics. The DRM core removes it when the device is +/// unregistered, and the caller keeps the snapshot alive until after that +/// registration is dropped, so the pointer stays valid for as long as the +/// file exists. +pub(crate) fn register(drm: &AsahiDrmDevice, stats: &Arc>) { + // `drm::Device` is `repr(C)` with the `drm_device` as its first field. + let raw = (drm as *const AsahiDrmDevice) + .cast_mut() + .cast::(); + // SAFETY: `raw` is the live DRM device. It is registered at this point, + // so its debugfs directory exists and the file is created in it. + unsafe { + bindings::drm_debugfs_add_file( + raw, + c"asahi_stats".as_char_ptr(), + Some(show), + Arc::as_ptr(stats).cast_mut().cast::(), + ); + } +} + +/// `drm_debugfs_add_file()` show callback. +/// +/// The DRM core opens the file with `single_open(file, node->show, entry)`, so +/// the seq file's private data is the `struct drm_debugfs_entry` and the +/// per-device payload is in `entry->file.data` (see `drm_debugfs_entry_open()` +/// in `drivers/gpu/drm/drm_debugfs.c`). The `data` argument of the callback is +/// the seq file start token, not the payload. +unsafe extern "C" fn show( + m: *mut bindings::seq_file, + _data: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: The DRM core passes a valid seq file to the show callback. + let seq = unsafe { SeqFile::from_raw(m) }; + // SAFETY: The seq file was opened with the DRM entry as private data. + let entry = unsafe { (*m).private.cast::() }; + // SAFETY: `entry->file.data` is the payload pointer we registered. + let stats = unsafe { (*entry).file.data.cast::>() }; + + let mut out = SeqWriter { seq }; + // SAFETY: The device-owned snapshot outlives its debugfs entry. + let snapshot = *unsafe { (*stats).lock() }; + let device = device_name(entry.cast_const()); + let _ = format_snapshot(&snapshot, monotonic_ns(), device, &mut out); + 0 +} + +/// Name of the DRM device the statistics belong to. +/// +/// The name is owned by the DRM device, which outlives the read. +fn device_name<'a>(entry: *const bindings::drm_debugfs_entry) -> &'a str { + // SAFETY: The DRM core passes the entry it allocated for this file. + let drm = unsafe { (*entry).dev }; + if drm.is_null() { + return "unknown"; + } + // SAFETY: `drm` is the live DRM device of this entry. + let unique = unsafe { (*drm).unique }; + if unique.is_null() { + return "unknown"; + } + // SAFETY: `unique` is a NUL terminated string owned by the DRM device, + // which outlives this read. + let name = unsafe { core::ffi::CStr::from_ptr(unique) }; + name.to_str().unwrap_or("unknown") +} + +/// [`Write`] adapter that forwards to a seq file. +struct SeqWriter<'a> { + seq: &'a SeqFile, +} + +impl Write for SeqWriter<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + self.seq.call_printf(fmt!("{}", value)); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/tests/test_asahi_stats_export.py b/drivers/gpu/drm/asahi/tests/test_asahi_stats_export.py new file mode 100755 index 0000000000000..b63cfbfccb3ba --- /dev/null +++ b/drivers/gpu/drm/asahi/tests/test_asahi_stats_export.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Exercise the raw firmware statistics snapshot and its export format. + +The snapshot keeps the firmware words verbatim: no unit is assumed, nothing is +scaled and no metric is derived. This harness pins that contract, because a +consumer parses the exported text. +""" + +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +stats = (ROOT / "stats_raw.rs").read_text() + + +def block(marker): + """Extract the item starting at `marker`, up to its matching brace.""" + start = stats.index(marker) + index = stats.index("{", start) + 1 + depth = 1 + while depth: + depth += (stats[index] == "{") - (stats[index] == "}") + index += 1 + return stats[start:index] + +format_tag = re.search(r"^pub\(crate\) const STATS_FORMAT: &str = \".*\";$", stats, re.MULTILINE) +assert format_tag is not None + + +source = "\n".join([ + "#![allow(dead_code, unused_variables)]", + "use std::fmt::Write;", + format_tag.group(), + block("#[derive(Copy, Clone, PartialEq, Eq)]"), + block("impl StatsKind"), + block("#[derive(Copy, Clone, Default)]"), + block("impl RawStatsSnapshot"), + block("pub(crate) fn format_snapshot"), + r''' +fn render(snap: &RawStatsSnapshot) -> String { + let mut out = String::new(); + format_snapshot(snap, 1000, "testdev", &mut out).unwrap(); + out +} + +const EMPTY: &str = "\ +asahi_stats_v1 +device=testdev +now_ns=1000 +messages count=0 +power count=0 +power_on count=0 +power_off count=0 +utilization count=0 +avg_power count=0 +temperature count=0 +power_state count=0 +fw_busy count=0 +pstate count=0 +temp_sensor count=0 +"; + +#[test] +fn missing_samples_are_explicit() { + let snap = RawStatsSnapshot::default(); + assert_eq!(render(&snap), EMPTY); + // A kind that was never received carries no payload at all, so a missing + // sample can never be read as a zero value. + for line in EMPTY.lines() { + if line.ends_with("count=0") { + assert_eq!(line.split_whitespace().count(), 2, "{}", line); + } + } +} + +#[test] +fn raw_samples_are_preserved_verbatim() { + let mut snap = RawStatsSnapshot::default(); + snap.note_message(); + snap.set_power(0x1234, 11); + snap.note_message(); + snap.set_power_on(0x2222, 12); + snap.note_message(); + snap.set_power_off(0x3333, 13); + snap.note_message(); + snap.set_utilization(0x4444, [1, 2, 3, 4], 14); + snap.note_message(); + snap.set_avg_power(0x5555, [6, 7, 8], 9, 15); + snap.note_message(); + snap.set_temperature(30000, 1000, 0, 100000, 16); + snap.note_message(); + snap.set_power_state(0x6666, 0x7777, 1, 0, 2, 5, 3, 4, 17); + snap.note_message(); + snap.set_fw_busy(0x8888, 1, 18); + snap.note_message(); + snap.set_pstate(0, 1, 7, 2, 19); + snap.note_message(); + snap.set_temp_sensor(2, 30000, 1000, 0, 100000, 20); + + assert_eq!(render(&snap), "\ +asahi_stats_v1 +device=testdev +now_ns=1000 +messages count=10 +power count=1 captured_ns=11 power=4660 +power_on count=1 captured_ns=12 off_time=8738 +power_off count=1 captured_ns=13 on_time=13107 +utilization count=1 captured_ns=14 timestamp=17476 util1=1 util2=2 util3=3 util4=4 +avg_power count=1 captured_ns=15 active_cs=21845 unk2=6 unk3=7 unk4=8 avg_power=9 +temperature count=1 captured_ns=16 raw_value=30000 scale=1000 tmin=0 tmax=100000 +power_state count=1 captured_ns=17 timestamp=26214 last_busy_ts=30583 active=1 poweroff=0 unk1=2 pstate=5 unk2=3 unk3=4 +fw_busy count=1 captured_ns=18 timestamp=34952 busy=1 +pstate count=1 captured_ns=19 ps_min=0 unk1=1 ps_max=7 unk2=2 +temp_sensor count=1 captured_ns=20 sensor_id=2 raw_value=30000 scale=1000 tmin=0 tmax=100000 +"); +} + +#[test] +fn counters_and_capture_times_track_samples() { + let mut snap = RawStatsSnapshot::default(); + snap.set_fw_busy(1, 1, 100); + snap.set_fw_busy(2, 0, 200); + let text = render(&snap); + assert!(text.contains("fw_busy count=2 captured_ns=200 timestamp=2 busy=0\n"), "{}", text); + // The sample count of a kind that was never received stays at zero. + assert!(text.contains("power count=0\n"), "{}", text); + assert!(text.contains("messages count=0\n"), "{}", text); +} + +#[test] +fn no_metric_semantics_are_invented() { + let mut snap = RawStatsSnapshot::default(); + snap.set_fw_busy(1, 1, 100); + snap.set_utilization(1, [10, 20, 30, 40], 100); + let text = render(&snap); + // The firmware busy flag is not a GPU utilisation figure and the raw + // counters keep their own names; no scaled or renamed metric appears. + for forbidden in ["gpu_busy", "util_pct", "utilization_pct", "power_mw", "temp_c", "freq_mhz"] { + assert!(!text.contains(forbidden), "{} in {}", forbidden, text); + } + assert!(text.contains("fw_busy count=1 captured_ns=100 timestamp=1 busy=1\n"), "{}", text); +} +''', +]) + +with tempfile.TemporaryDirectory(prefix="asahi-stats-export-") as tmp: + source_path = Path(tmp) / "test.rs" + binary = Path(tmp) / "test" + source_path.write_text(source) + subprocess.run(["rustc", "--edition=2021", "--test", str(source_path), + "-o", str(binary)], check=True) + subprocess.run([str(binary)], check=True) diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 145711b2ccf00..6a8789909bbc8 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include From 8150f30fd7209c0ddb29c76d0213506d71880169 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:54:05 -0500 Subject: [PATCH 35/46] Documentation/ABI: describe the asahi_stats debugfs file The raw firmware statistics export had no format document. This adds the ABI entry for the asahi_stats_v1 text format: file location, line order, fields, and what is not established. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> (cherry picked from commit 2d2ac34361918c1075e33e888d5a9738e4642da9) Assisted-by: LLM Signed-off-by: Ryan Murray --- .../ABI/testing/debugfs-driver-asahi-stats | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 Documentation/ABI/testing/debugfs-driver-asahi-stats diff --git a/Documentation/ABI/testing/debugfs-driver-asahi-stats b/Documentation/ABI/testing/debugfs-driver-asahi-stats new file mode 100644 index 0000000000000..dae8fbebb3db8 --- /dev/null +++ b/Documentation/ABI/testing/debugfs-driver-asahi-stats @@ -0,0 +1,47 @@ +What: /sys/kernel/debug/dri//asahi_stats +Date: October 2026 +Contact: Asahi Linux GPU driver maintainers +Description: + Read-only, root-only dump of the raw AGX firmware statistics + that the drm/asahi driver receives on its statistics channel. + The driver stores the latest message of each kind and prints + it unchanged. It does not compute utilization or any other + derived value. + + The file is created in the debugfs directory of the DRM + device when the device is registered. It also appears under + the other names of that directory (dri/128 and the platform + device name, for example dri/406400000.gpu). + + The format is ASCII text, one line per record: + + - line 1: asahi_stats_v1, the format tag. A reader must check it. + - line 2: device= + - line 3: now_ns= + - line 4: messages count= + - then one line per message kind, in this order: power, + power_on, power_off, utilization, avg_power, temperature, + power_state, fw_busy, pstate, temp_sensor. + + A kind line is " count=" followed, when n is not 0, + by " captured_ns=" and the fields of + that kind as key=value pairs: + + - power: power + - power_on: off_time + - power_off: on_time + - utilization: timestamp, util1, util2, util3, util4 + - avg_power: active_cs, unk2, unk3, unk4, avg_power + - temperature: raw_value, scale, tmin, tmax + - power_state: timestamp, last_busy_ts, active, poweroff, + unk1, pstate, unk2, unk3 + - fw_busy: timestamp, busy + - pstate: ps_min, unk1, ps_max, unk2 + - temp_sensor: sensor_id, raw_value, scale, tmin, tmax + + All counts are cumulative since boot. A rate needs two reads. + The values are the raw firmware values. Their units and the + meaning of the util*, unk* fields are not established. A kind + that was never received prints count=0 and no payload. Keys + may be added in later versions of the format tag; readers + must ignore keys they do not know. From ffeecc8cc5fe2db09ecb2e7795dbd009e87dcc75 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:08:07 -0500 Subject: [PATCH 36/46] Documentation/ABI: fix stats counter wording and contact line Counts restart when the driver binds the device again, so say that instead of since boot. Use the public maintainer name in Contact. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> (cherry picked from commit c59f572e0aa69a0c34383644d5deb5ee3f79e60f) [ryan: Contact is the maintainer named in our other Aurora ABI files, not omacom's] Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/ABI/testing/debugfs-driver-asahi-stats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/ABI/testing/debugfs-driver-asahi-stats b/Documentation/ABI/testing/debugfs-driver-asahi-stats index dae8fbebb3db8..f795d25ace2ba 100644 --- a/Documentation/ABI/testing/debugfs-driver-asahi-stats +++ b/Documentation/ABI/testing/debugfs-driver-asahi-stats @@ -1,6 +1,6 @@ What: /sys/kernel/debug/dri//asahi_stats Date: October 2026 -Contact: Asahi Linux GPU driver maintainers +Contact: Ryan Murray Description: Read-only, root-only dump of the raw AGX firmware statistics that the drm/asahi driver receives on its statistics channel. @@ -39,7 +39,7 @@ Description: - pstate: ps_min, unk1, ps_max, unk2 - temp_sensor: sensor_id, raw_value, scale, tmin, tmax - All counts are cumulative since boot. A rate needs two reads. + All counts are cumulative since the driver bound the device. A rate needs two reads. The values are the raw firmware values. Their units and the meaning of the util*, unk* fields are not established. A kind that was never received prints count=0 and no payload. Keys From 553f083f3b256e2209d7041b6599d67c128a753c Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:11:53 -0500 Subject: [PATCH 37/46] drm/asahi: only create the stats file with CONFIG_DEBUG_FS Without CONFIG_DEBUG_FS drm_debugfs_add_file() is a static inline stub, so bindgen emits no binding and the Rust build fails. The KUnit configuration has debugfs off, which is how the CI KUnit job found it. Create the file only when CONFIG_DEBUG_FS is set. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> (cherry picked from commit ad967fa58fb7f649ddcb445bf156f2eae0e52e45) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/stats_raw.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/asahi/stats_raw.rs b/drivers/gpu/drm/asahi/stats_raw.rs index 8d5de5bf59607..3d158337f66b7 100644 --- a/drivers/gpu/drm/asahi/stats_raw.rs +++ b/drivers/gpu/drm/asahi/stats_raw.rs @@ -424,6 +424,9 @@ pub(crate) fn register(drm: &AsahiDrmDevice, stats: &Arc(); + // `drm_debugfs_add_file()` is a static inline stub without CONFIG_DEBUG_FS, so + // bindgen has no binding for it and there is no file to create. + #[cfg(CONFIG_DEBUG_FS)] // SAFETY: `raw` is the live DRM device. It is registered at this point, // so its debugfs directory exists and the file is created in it. unsafe { @@ -434,6 +437,8 @@ pub(crate) fn register(drm: &AsahiDrmDevice, stats: &Arc(), ); } + #[cfg(not(CONFIG_DEBUG_FS))] + let _ = (raw, stats); } /// `drm_debugfs_add_file()` show callback. @@ -443,6 +448,7 @@ pub(crate) fn register(drm: &AsahiDrmDevice, stats: &Arcfile.data` (see `drm_debugfs_entry_open()` /// in `drivers/gpu/drm/drm_debugfs.c`). The `data` argument of the callback is /// the seq file start token, not the payload. +#[cfg_attr(not(CONFIG_DEBUG_FS), allow(dead_code))] unsafe extern "C" fn show( m: *mut bindings::seq_file, _data: *mut core::ffi::c_void, From 14432bc409915206854f2c526135f41357f8b120 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:25:17 +1100 Subject: [PATCH 38/46] tools/asahi: run the raw statistics export test with the host tests The asahi_stats format test came in as drivers/gpu/drm/asahi/tests/ test_asahi_stats_export.py, which nothing runs and which has no SPDX line. Move it to tools/asahi/asahi-stats-raw-tests.py, read stats_raw.rs from the tree root, add the SPDX line and register it in tools/asahi/run-host-tests.py so it runs with every host test pass. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../asahi/asahi-stats-raw-tests.py | 5 +++-- tools/asahi/run-host-tests.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) rename drivers/gpu/drm/asahi/tests/test_asahi_stats_export.py => tools/asahi/asahi-stats-raw-tests.py (97%) diff --git a/drivers/gpu/drm/asahi/tests/test_asahi_stats_export.py b/tools/asahi/asahi-stats-raw-tests.py similarity index 97% rename from drivers/gpu/drm/asahi/tests/test_asahi_stats_export.py rename to tools/asahi/asahi-stats-raw-tests.py index b63cfbfccb3ba..1dc45655eb548 100755 --- a/drivers/gpu/drm/asahi/tests/test_asahi_stats_export.py +++ b/tools/asahi/asahi-stats-raw-tests.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT """Exercise the raw firmware statistics snapshot and its export format. The snapshot keeps the firmware words verbatim: no unit is assumed, nothing is @@ -11,8 +12,8 @@ import subprocess import tempfile -ROOT = Path(__file__).resolve().parents[1] -stats = (ROOT / "stats_raw.rs").read_text() +ROOT = Path(__file__).resolve().parents[2] +stats = (ROOT / "drivers/gpu/drm/asahi/stats_raw.rs").read_text() def block(marker): diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 15f52ce90ffb6..a075e2a1dab20 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -64,6 +64,7 @@ def mutations(flag, names, base, expect=KILL): 'apple-pwm-pm-tests.py': [([], PASS), (['--negative-control'], PASS)], 'apple-dart-fw-tests.py': [(out(), PASS)], 'apple-dart-range-tests.py': [([], PASS)], + 'asahi-stats-raw-tests.py': [([], PASS)], 'atc-usb3-tests.py': [(out(), PASS), (out() + ['--mutations'], PASS)], 'avd-av1-tail-tests.py': [(out(), PASS)] + [(out() + ['--case', c], PASS) for c in ('copy', 'seed', 'aligned')] From 1761ce507dcf434fda8131b5b99ba8e231b98dd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20Carre=C3=B1o?= Date: Sun, 25 Oct 2026 08:16:21 +0100 Subject: [PATCH 39/46] drm/asahi: g15: renumber the TA/FS stamps before the TA stamp wraps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TA stamp of render ordinal n is stamp_seed(TA_SEED, n): it steps by 0x100 and wraps after 0x860000 renders, about six hours of desktop. With the stamp_seed() from before commit 7ccdfe85843d ("drm/asahi: wrap G15 stamps through zero"), which skipped zero and went from 0xffffff00 back to 0x100 there, the firmware stopped at render 8781824 with "DAG: Non Sequential Stamp Updates seen ... stampValue 0x100" and the GPU was lost until reboot. Since that commit the stamps wrap through zero; whether the firmware takes that wrap has not been tried. Add an offset to the render ordinal used for the TA/FS stamps and, when g15_stamp_reseed_margin is set, renumber both stamps from their seeds at the first publication with nothing in flight inside that margin: write the seeds to the user and firmware stamps, reset the engine targets and move the offset so the next render counts as ordinal 1, as after boot. The offset is taken modulo the 2^24 ordinals after which stamp_seed() repeats (STAMP_PERIOD). The stamps then never reach the wrap. g15_stamp_base starts the count near the wrap, to test this in seconds instead of hours. Both parameters default to 0, which keeps the stamps exactly as before. Tested on a J516C, together with the next commit, on integration/aurora with the earlier stamp_seed() and the offset taken modulo its period of 0xffffff ordinals: from g15_stamp_base 2000 renders before the wrap with a margin of 1000, the stamps were renumbered at render 1000 and 3003 renders retired; in desktop use, one boot renumbered at renders 8681824 and 17363648. On this base, modulo 0xffffff would give the render after the first renumbering ordinal 0, whose targets the seeds already meet, and the one after the second ordinal -1, a TA target below the seed. Generated-by: LLM Signed-off-by: Samuel Carreño (cherry picked from commit 541f3e46b3520927cd545231fb081de50c6d145e) [ryan: read the two stamp parameters by value, as the Linux 7.3 module parameter API returns them] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/asahi_core.rs | 8 +++++ drivers/gpu/drm/asahi/g15/encode.rs | 8 +++++ drivers/gpu/drm/asahi/g15/engine.rs | 52 +++++++++++++++++++++++++---- drivers/gpu/drm/asahi/g15/graph.rs | 5 +-- drivers/gpu/drm/asahi/g15/params.rs | 12 +++++++ 5 files changed, 76 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 1e0b043b8b183..0b2c5946dd56e 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -189,6 +189,14 @@ module! { default: 1024, description: "G15: PB blocks posted above which an idle engine reclaims the PB", }, + g15_stamp_base: u32 { + default: 0, + description: "G15 (testing): render ordinal the TA/FS stamps start from; 0 = unchanged", + }, + g15_stamp_reseed_margin: u32 { + default: 0, + description: "G15: renders before the TA stamp wrap within which TA/FS are renumbered while idle; 0 = off", + }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", diff --git a/drivers/gpu/drm/asahi/g15/encode.rs b/drivers/gpu/drm/asahi/g15/encode.rs index 64aebf3254891..4715fe873ba29 100644 --- a/drivers/gpu/drm/asahi/g15/encode.rs +++ b/drivers/gpu/drm/asahi/g15/encode.rs @@ -69,6 +69,14 @@ pub(crate) const fn stamp_seed(seed: u32, n: u64) -> u32 { seed.wrapping_add((n as u32).wrapping_mul(0x100)) } +/// Ordinals after which [`stamp_seed`] repeats: 2^32 / 0x100. +pub(crate) const STAMP_PERIOD: u64 = 1 << 24; + +/// Effective render ordinal at which the TA stamp wraps (0xffffff00 -> 0): 0x860000 with the TA +/// seed. A wrap that skipped zero stopped the firmware ("DAG: Non Sequential Stamp Updates"); +/// the renumbering keeps the stamps away from it. The FS stamp wraps later (0xc30000). +pub(crate) const RENDER_WRAP: u64 = 0xffffff - (hw::engine::TA_SEED / 0x100) as u64 + 1; + /// Compute stamp of ordinal `n`. pub(crate) const fn stamp(n: u64) -> u32 { stamp_seed(hw::engine::COMPUTE_SEED, n) diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 754f5eac4ff36..a8d0af56bd845 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -364,6 +364,9 @@ pub(crate) struct Engine { /// Host compute ordinal and draw ordinal of the last publication. pub(crate) ordinal: u64, pub(crate) draw: u64, + /// Added to the draw ordinal before computing the TA/FS stamps; moved by + /// `renumber_render_stamps()`. + pub(crate) stamp_base: u64, /// Newest published receipts, the tail. pub(crate) target: u32, pub(crate) head: u32, @@ -411,6 +414,7 @@ impl Engine { shared: None, ordinal: 0, draw: 0, + stamp_base: params::stamp_base(), target: 0, head: 0, ta_target: 0, @@ -708,6 +712,37 @@ impl Firmware { result } + /// Renumbers the TA/FS stamps from their seeds before the TA stamp wraps, after 0x860000 + /// renders (about six hours of desktop), where the firmware may stop. Only called with + /// nothing in flight: the last flight retired with every stamp equal to its target, so both + /// queues are idle and the next render waits on exactly the seeds written here. + fn renumber_render_stamps(&mut self) -> Result { + let margin = params::stamp_reseed_margin(); + let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; + if margin == 0 || ordinal + margin < encode::RENDER_WRAP { + return Ok(()); + } + let r = &self.graph.render; + for (e, seed) in [ids::TA_SEED, ids::FS_SEED].into_iter().enumerate() { + r.user_stamp[e].w32(0, seed)?; + r.fw_stamp[e].w32(0, seed)?; + } + // The next render (draw + 1) counts as ordinal 1 again: modulo the period of + // `stamp_seed`, whose ordinal 0 is the seeds just written. + let period = encode::STAMP_PERIOD; + self.engine.stamp_base = (period - self.engine.draw % period) % period; + self.engine.ta_target = ids::TA_SEED; + self.engine.fs_target = ids::FS_SEED; + self.engine.tail_fs = ids::FS_SEED; + dev_info!( + self.graph.dev.as_ref(), + "G15: TA/FS stamps renumbered from the seeds at draw {} (ordinal {:#x})\n", + self.engine.draw, + ordinal + ); + Ok(()) + } + fn publish_prefix_into(&mut self, job: &Arc, f: &mut Flight) -> Result { // The PB backing is mapped into the job's VM before anything of the prefix is changed. // Nothing is published when that fails, so the error must not be EIO (which reports a @@ -715,6 +750,9 @@ impl Firmware { self.map_pb(&job.vm) .map_err(|e| if e == EIO { EFAULT } else { e })?; let overlap = !self.engine.flights.is_empty(); + if !overlap { + self.renumber_render_stamps()?; + } let mut mixed = tuning::MIXED_PREFIX; let mut pf = Prefix { n: self.engine.ordinal, @@ -986,7 +1024,7 @@ impl Firmware { self.engine.head = pf.c_head; } if renders != 0 { - f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw); + f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw + self.engine.stamp_base); self.engine.ta_target = f.ta_target; f.fs_target = pf.last_fs; self.engine.fs_target = pf.last_fs; @@ -1265,8 +1303,9 @@ impl Firmware { let n = pf.draw + 1; let usc = job.queue.usc_base; let ctx = job.slot(); - let tta = encode::stamp_seed(ids::TA_SEED, n); - let tfs = encode::stamp_seed(ids::FS_SEED, n); + let sn = n + self.engine.stamp_base; + let tta = encode::stamp_seed(ids::TA_SEED, sn); + let tfs = encode::stamp_seed(ids::FS_SEED, sn); // Early tiling: a later render member after a render of the same prefix whose render // dependencies are fragment-only (FRAGMENT_BARRIERS) and that has no start timestamps // lets its TA wait for the previous TA; its FS still waits for the previous FS, then @@ -1365,7 +1404,7 @@ impl Firmware { } else if early { let b = encode::barrier( r.fw_stamp[0].va, - encode::stamp_seed(ids::TA_SEED, n - 1), + encode::stamp_seed(ids::TA_SEED, sn - 1), 0, tta, RENDER_WAIT_UUID, @@ -1385,10 +1424,9 @@ impl Firmware { } else { let mut prev_fs = if pf.prev != Prev::None { pf.last_fs - } else if n == 1 { - ids::FS_SEED } else { - encode::stamp_seed(ids::FS_SEED, n - 1) + // With no offset, ordinal 0 is the FS seed itself (the first render). + encode::stamp_seed(ids::FS_SEED, sn - 1) }; // Restricted type 0: predecessor render in this prefix, no InitBM/resync. let internal = pf.prev == Prev::Render diff --git a/drivers/gpu/drm/asahi/g15/graph.rs b/drivers/gpu/drm/asahi/g15/graph.rs index 9a62f25f19a63..e7d1471eb2d88 100644 --- a/drivers/gpu/drm/asahi/g15/graph.rs +++ b/drivers/gpu/drm/asahi/g15/graph.rs @@ -352,8 +352,9 @@ impl RenderGraph { q.w64(qinfo::UNK_AC, 4)?; self.qstate[e].w32(qstate::CAP, sizes::QUEUE_CAP)?; self.ring[e].fill(0, 0xcc, sizes::QUEUE_CAP as usize * 8)?; - self.user_stamp[e].w32(0, seed[e])?; - self.fw_stamp[e].w32(0, seed[e])?; + let start = super::encode::stamp_seed(seed[e], super::params::stamp_base()); + self.user_stamp[e].w32(0, start)?; + self.fw_stamp[e].w32(0, start)?; } init_sched_ctx(&self.sched_ctx)?; init_notifier(&self.notifier, &self.threshold)?; diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index 7589e25aa4277..fecbf167aa3f4 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -18,6 +18,18 @@ use crate::module_parameters; /// updates); this bounds the kernel's staging allocation. pub(crate) const MAX_BINDS: u32 = 1 << 20; +/// Render ordinal the TA/FS stamps start from. Only for testing the stamp wrap without hours of +/// drawing; 0 keeps the stamps exactly as before. +pub(crate) fn stamp_base() -> u64 { + (module_parameters::g15_stamp_base.value() as u64) % super::encode::STAMP_PERIOD +} + +/// Renders before the TA stamp wrap within which the TA/FS stamps are renumbered from their seeds, +/// at the first moment with nothing in flight (0: never). +pub(crate) fn stamp_reseed_margin() -> u64 { + module_parameters::g15_stamp_reseed_margin.value() as u64 +} + /// Host deadline of a published prefix without progress, ms (at least 100). pub(crate) fn deadline_ms() -> u32 { module_parameters::g15_deadline_ms.value().max(100) From 433c2871ce31c4388837fdd1c3b76fb52cdf5793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20Carre=C3=B1o?= Date: Sun, 25 Oct 2026 15:15:52 +0100 Subject: [PATCH 40/46] drm/asahi: g15: drain the flights inside the stamp renumbering margin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renumbering only happens at a publication with nothing in flight. A client that keeps the GPU busy (no vsync) can keep a flight in the air through the whole margin, and the TA stamp would then wrap as before. Inside the margin, publish nothing behind a flight, so the flights drain and the next publication renumbers while idle (once every ~8.7M renders with a margin of 100000). Also clamp a non-zero margin to at least one prefix of renders, so that the last prefix published outside it cannot cross the wrap, and to at most half the stamp range, so that it cannot renumber on every publication. Found while reviewing the previous commit, with a model of prefixes of 1-8 renders: a fully overlapped client wrapped 199/199 times without this and 0/199 with it. On a J516C, with the earlier stamp_seed() (see the previous commit), from g15_stamp_base 500000 renders before the wrap with a margin of 100000 and the GPU kept busy (286875 publications behind a flight), the stamps were renumbered at render 400000 and rendering went on past the wrap point to render 526113. Generated-by: LLM Signed-off-by: Samuel Carreño (cherry picked from commit 6f7e3df42fab1b281920405cb9faab0b9affeddc) [ryan: keep reading the margin parameter by value, as on Linux 7.3] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/engine.rs | 17 ++++++++++++++--- drivers/gpu/drm/asahi/g15/params.rs | 12 ++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index a8d0af56bd845..52ef8cc9eea05 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -564,12 +564,17 @@ impl Firmware { break; }; let job = self.engine.pending[index].clone(); + // Inside the renumbering margin nothing is published behind a flight: the flights + // drain and the next publication renumbers the stamps while idle (once every ~8.7M + // renders with a margin of 100000). + let renumber_due = self.renumber_due(); if let Some(tail) = self.engine.flights.last() { if self.engine.flights.len() >= max_inflight || !Arc::ptr_eq(&job.vm, &tail.job.vm) || self.recovery.reinit || self.recovery.fs_resync || self.engine.rebuild_live + || renumber_due { self.engine.stats.hold[0] += (self.engine.flights.len() >= max_inflight) as u64; break; @@ -712,16 +717,22 @@ impl Firmware { result } + /// The TA stamp is inside its renumbering margin. + fn renumber_due(&self) -> bool { + let margin = params::stamp_reseed_margin(); + let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; + margin != 0 && ordinal + margin >= encode::RENDER_WRAP + } + /// Renumbers the TA/FS stamps from their seeds before the TA stamp wraps, after 0x860000 /// renders (about six hours of desktop), where the firmware may stop. Only called with /// nothing in flight: the last flight retired with every stamp equal to its target, so both /// queues are idle and the next render waits on exactly the seeds written here. fn renumber_render_stamps(&mut self) -> Result { - let margin = params::stamp_reseed_margin(); - let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; - if margin == 0 || ordinal + margin < encode::RENDER_WRAP { + if !self.renumber_due() { return Ok(()); } + let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; let r = &self.graph.render; for (e, seed) in [ids::TA_SEED, ids::FS_SEED].into_iter().enumerate() { r.user_stamp[e].w32(0, seed)?; diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index fecbf167aa3f4..31739c685c07b 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -25,9 +25,17 @@ pub(crate) fn stamp_base() -> u64 { } /// Renders before the TA stamp wrap within which the TA/FS stamps are renumbered from their seeds, -/// at the first moment with nothing in flight (0: never). +/// at the first moment with nothing in flight (0: never). At least one prefix of renders, so +/// that the last prefix published outside the margin cannot cross the wrap; at most half the +/// stamp range. pub(crate) fn stamp_reseed_margin() -> u64 { - module_parameters::g15_stamp_reseed_margin.value() as u64 + match module_parameters::g15_stamp_reseed_margin.value() as u64 { + 0 => 0, + m => m.clamp( + hw::tuning::PREFIX_RENDERS as u64, + super::encode::RENDER_WRAP / 2, + ), + } } /// Host deadline of a published prefix without progress, ms (at least 100). From 69617c1a556cf65fe1984b29838a4282d9a60239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20Carre=C3=B1o?= Date: Thu, 29 Oct 2026 12:23:58 +0100 Subject: [PATCH 41/46] drm/asahi: g15: renumber the compute stamp before it wraps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compute stamp of ordinal n is stamp_seed(COMPUTE_SEED, n): it steps by 0x100 from 0xc1000000 and, after 0x3f0000 computes (4128768: about 390 FLUX.3 1024x1024 images at one node per submission), goes from 0xffffff00 back to 0. Whether the firmware takes that wrap has not been tried either; the TA/FS renumbering of the previous commits does not cover it. Renumber the compute stamp the same way. Add an offset to the compute ordinal used for the stamp and, when g15_compute_reseed_margin is set, at the first publication with nothing in flight inside that margin write the compute stamps back as ComputeGraph::init() leaves them with no stamp base (user COMPUTE_SEED, firmware 0), reset the compute target and move the offset so the next compute counts as ordinal 1. Inside the margin nothing is published behind a flight, as for TA/FS, so the flights drain; the margin is at least one prefix of computes (COMPUTE_SLOTS) and at most half the stamp range. The ordinal itself, which the compute image also carries, keeps counting as the render draw does. g15_compute_stamp_base starts the count near the wrap, to test this in seconds instead of hours. Both parameters default to 0, which keeps the stamp exactly as before. On a J516C, with g15_compute_stamp_base=4023768 (105000 computes before the wrap) and g15_compute_reseed_margin=100000, the stamp was renumbered at compute 5000, the GPU kept running computes up to 159899, past the point where the stamp would have wrapped, and a FLUX.3 1024x1024 image generated fine afterwards. Generated-by: LLM Signed-off-by: Samuel Carreño (cherry picked from commit 736063aa051282b01eab9bae5af13f5481e24841) [ryan: read the two compute stamp parameters by value, as the Linux 7.3 module parameter API returns them] Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/asahi_core.rs | 8 +++++ drivers/gpu/drm/asahi/g15/encode.rs | 11 ++++-- drivers/gpu/drm/asahi/g15/engine.rs | 55 ++++++++++++++++++++++++++--- drivers/gpu/drm/asahi/g15/graph.rs | 3 +- drivers/gpu/drm/asahi/g15/params.rs | 20 +++++++++++ 5 files changed, 88 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 0b2c5946dd56e..d5daef485f06a 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -197,6 +197,14 @@ module! { default: 0, description: "G15: renders before the TA stamp wrap within which TA/FS are renumbered while idle; 0 = off", }, + g15_compute_stamp_base: u32 { + default: 0, + description: "G15 (testing): compute ordinal the compute stamp starts from; 0 = unchanged", + }, + g15_compute_reseed_margin: u32 { + default: 0, + description: "G15: computes before the compute stamp wrap within which it is renumbered while idle; 0 = off", + }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", diff --git a/drivers/gpu/drm/asahi/g15/encode.rs b/drivers/gpu/drm/asahi/g15/encode.rs index 4715fe873ba29..a62cca8613b33 100644 --- a/drivers/gpu/drm/asahi/g15/encode.rs +++ b/drivers/gpu/drm/asahi/g15/encode.rs @@ -77,6 +77,10 @@ pub(crate) const STAMP_PERIOD: u64 = 1 << 24; /// the renumbering keeps the stamps away from it. The FS stamp wraps later (0xc30000). pub(crate) const RENDER_WRAP: u64 = 0xffffff - (hw::engine::TA_SEED / 0x100) as u64 + 1; +/// Compute ordinal at which the compute stamp wraps (0xffffff00 -> 0): 0x3f0000 (4128768 +/// computes) with the compute seed, where the firmware may stop as at the TA stamp wrap. +pub(crate) const COMPUTE_WRAP: u64 = STAMP_PERIOD - (hw::engine::COMPUTE_SEED / 0x100) as u64; + /// Compute stamp of ordinal `n`. pub(crate) const fn stamp(n: u64) -> u32 { stamp_seed(hw::engine::COMPUTE_SEED, n) @@ -109,8 +113,9 @@ pub(crate) struct ComputeRefs { pub(crate) scratch: [u64; 5], } -/// Encodes compute command `c` as ordinal `n`. The engine copies the result into the slot -/// after zeroing the slot's images. +/// Encodes compute command `c` as ordinal `n` with stamp `t` (the engine's renumbering moves +/// the stamp off [`stamp`]`(n)`). The engine copies the result into the slot after zeroing the +/// slot's images. #[allow(clippy::too_many_arguments)] pub(crate) fn encode_compute( r: &ComputeRefs, @@ -120,12 +125,12 @@ pub(crate) fn encode_compute( ts_va: &[u64; 2], ctx: u32, n: u64, + t: u32, fault_mask: u32, w: &mut [u8; compute::WORK_SIZE], q: &mut [u8; compute::SEQ_SIZE], ) { let (wv, qv, h, pre) = (r.work, r.qinfo, r.hwdata, r.preempt_gva); - let t = stamp(n); let att_count = att.len().min(hw::sizes::MAX_ATTACHMENTS); // The saved-state pointers into the preemption buffer and the two 0x1c words; a SoC // encoded "fresh" carries zero in all six, as a first command does. diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 52ef8cc9eea05..76177470fea1c 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -367,6 +367,9 @@ pub(crate) struct Engine { /// Added to the draw ordinal before computing the TA/FS stamps; moved by /// `renumber_render_stamps()`. pub(crate) stamp_base: u64, + /// Added to the compute ordinal before computing the compute stamp; moved by + /// `renumber_compute_stamps()`. + pub(crate) compute_stamp_base: u64, /// Newest published receipts, the tail. pub(crate) target: u32, pub(crate) head: u32, @@ -415,6 +418,7 @@ impl Engine { ordinal: 0, draw: 0, stamp_base: params::stamp_base(), + compute_stamp_base: params::compute_stamp_base(), target: 0, head: 0, ta_target: 0, @@ -564,9 +568,9 @@ impl Firmware { break; }; let job = self.engine.pending[index].clone(); - // Inside the renumbering margin nothing is published behind a flight: the flights + // Inside a renumbering margin nothing is published behind a flight: the flights // drain and the next publication renumbers the stamps while idle (once every ~8.7M - // renders with a margin of 100000). + // renders or ~4.0M computes with margins of 100000). let renumber_due = self.renumber_due(); if let Some(tail) = self.engine.flights.last() { if self.engine.flights.len() >= max_inflight @@ -718,18 +722,30 @@ impl Firmware { } /// The TA stamp is inside its renumbering margin. - fn renumber_due(&self) -> bool { + fn render_renumber_due(&self) -> bool { let margin = params::stamp_reseed_margin(); let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; margin != 0 && ordinal + margin >= encode::RENDER_WRAP } + /// The compute stamp is inside its renumbering margin. + fn compute_renumber_due(&self) -> bool { + let margin = params::compute_reseed_margin(); + let ordinal = (self.engine.ordinal + self.engine.compute_stamp_base) % encode::STAMP_PERIOD; + margin != 0 && ordinal + margin >= encode::COMPUTE_WRAP + } + + /// Either stamp is inside its renumbering margin. + fn renumber_due(&self) -> bool { + self.render_renumber_due() || self.compute_renumber_due() + } + /// Renumbers the TA/FS stamps from their seeds before the TA stamp wraps, after 0x860000 /// renders (about six hours of desktop), where the firmware may stop. Only called with /// nothing in flight: the last flight retired with every stamp equal to its target, so both /// queues are idle and the next render waits on exactly the seeds written here. fn renumber_render_stamps(&mut self) -> Result { - if !self.renumber_due() { + if !self.render_renumber_due() { return Ok(()); } let ordinal = (self.engine.draw + self.engine.stamp_base) % encode::STAMP_PERIOD; @@ -754,6 +770,33 @@ impl Firmware { Ok(()) } + /// Renumbers the compute stamp from its seed before it wraps, after 0x3f0000 computes, + /// where the firmware may stop. Like the TA/FS renumbering only with nothing in flight: the + /// compute queue is idle with both stamps at its target. The stamps are written back as + /// `ComputeGraph::init()` leaves them with no stamp base, so the next compute is ordinal 1 + /// on the stamps of a fresh boot. + fn renumber_compute_stamps(&mut self) -> Result { + if !self.compute_renumber_due() { + return Ok(()); + } + let ordinal = (self.engine.ordinal + self.engine.compute_stamp_base) % encode::STAMP_PERIOD; + let g = &self.graph.compute; + g.user_stamp.w32(0, ids::COMPUTE_SEED)?; + g.fw_stamp.w32(0, 0)?; + // The next compute (ordinal + 1) counts as ordinal 1 again. + let period = encode::STAMP_PERIOD; + self.engine.compute_stamp_base = (period - self.engine.ordinal % period) % period; + self.engine.target = 0; + self.engine.tail_ct = 0; + dev_info!( + self.graph.dev.as_ref(), + "G15: compute stamp renumbered from the seed at compute {} (ordinal {:#x})\n", + self.engine.ordinal, + ordinal + ); + Ok(()) + } + fn publish_prefix_into(&mut self, job: &Arc, f: &mut Flight) -> Result { // The PB backing is mapped into the job's VM before anything of the prefix is changed. // Nothing is published when that fails, so the error must not be EIO (which reports a @@ -763,6 +806,7 @@ impl Firmware { let overlap = !self.engine.flights.is_empty(); if !overlap { self.renumber_render_stamps()?; + self.renumber_compute_stamps()?; } let mut mixed = tuning::MIXED_PREFIX; let mut pf = Prefix { @@ -1171,7 +1215,7 @@ impl Firmware { later_render: bool, ) -> Result { let n = pf.n + 1; - let t = encode::stamp(n); + let t = encode::stamp(n + self.engine.compute_stamp_base); let ctx = job.slot(); let ts = &cmd.timestamps; let mut lap = Lap::start(); @@ -1215,6 +1259,7 @@ impl Firmware { ts, ctx, n, + t, tuning::COMPUTE_FAULT_MASK, image(w)?, image(q)?, diff --git a/drivers/gpu/drm/asahi/g15/graph.rs b/drivers/gpu/drm/asahi/g15/graph.rs index e7d1471eb2d88..18680fd05596f 100644 --- a/drivers/gpu/drm/asahi/g15/graph.rs +++ b/drivers/gpu/drm/asahi/g15/graph.rs @@ -251,7 +251,8 @@ impl ComputeGraph { q.w64(qinfo::UNK_AC, 4)?; self.qstate.w32(qstate::CAP, sizes::QUEUE_CAP)?; - self.user_stamp.w32(0, engine::COMPUTE_SEED)?; + let start = super::encode::stamp(super::params::compute_stamp_base()); + self.user_stamp.w32(0, start)?; self.fw_stamp.w32(0, 0) } } diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index 31739c685c07b..a5549c3010a6f 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -38,6 +38,26 @@ pub(crate) fn stamp_reseed_margin() -> u64 { } } +/// Compute ordinal the compute stamp starts from. Only for testing the stamp wrap without +/// millions of computes; 0 keeps the stamp exactly as before. +pub(crate) fn compute_stamp_base() -> u64 { + (module_parameters::g15_compute_stamp_base.value() as u64) % super::encode::STAMP_PERIOD +} + +/// Computes before the compute stamp wrap within which the compute stamp is renumbered from its +/// seed, at the first moment with nothing in flight (0: never). At least one prefix of computes, +/// so that the last prefix published outside the margin cannot cross the wrap; at most half the +/// stamp range. +pub(crate) fn compute_reseed_margin() -> u64 { + match module_parameters::g15_compute_reseed_margin.value() as u64 { + 0 => 0, + m => m.clamp( + hw::sizes::COMPUTE_SLOTS as u64, + super::encode::COMPUTE_WRAP / 2, + ), + } +} + /// Host deadline of a published prefix without progress, ms (at least 100). pub(crate) fn deadline_ms() -> u32 { module_parameters::g15_deadline_ms.value().max(100) From d8bd85f040fe0667a38d1df94589b8d5ff0d593d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20Carre=C3=B1o?= Date: Thu, 29 Oct 2026 12:29:14 +0100 Subject: [PATCH 42/46] drm/asahi: g15: show what is left before the stamps wrap in the stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a "stamps:" line to asahi-g15/stats with the computes and renders the current numbering has left before the compute and TA stamps wrap, and both renumbering margins (0: off). A script that limits compute work before the wrap can then tell a kernel that renumbers the compute stamp from one that does not, and count from the stamp rather than from boot (g15_compute_stamp_base moves the start). Generated-by: LLM Signed-off-by: Samuel Carreño (cherry picked from commit c3fad65647f0e46223b91fb7a562d2f1a7c3e945) Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/g15/engine.rs | 12 ++++++++++++ drivers/gpu/drm/asahi/g15/poll.rs | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 76177470fea1c..1a00f33e0c96b 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -740,6 +740,18 @@ impl Firmware { self.render_renumber_due() || self.compute_renumber_due() } + /// Computes and renders the current numbering has left before its compute and TA stamps + /// wrap (0 once wrapped), for `asahi-g15/stats`. + pub(crate) fn stamps_left(&self) -> (u64, u64) { + let period = encode::STAMP_PERIOD; + let c = (self.engine.ordinal + self.engine.compute_stamp_base) % period; + let r = (self.engine.draw + self.engine.stamp_base) % period; + ( + encode::COMPUTE_WRAP.saturating_sub(c), + encode::RENDER_WRAP.saturating_sub(r), + ) + } + /// Renumbers the TA/FS stamps from their seeds before the TA stamp wraps, after 0x860000 /// renders (about six hours of desktop), where the firmware may stop. Only called with /// nothing in flight: the last flight retired with every stamp equal to its target, so both diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 7d4e2b9dd51ba..282a09c4c4e11 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -737,6 +737,16 @@ impl Firmware { s.hz_ahead )?; self.pools.write_stats(f)?; + // What is left before the stamps wrap, and the renumbering margins (0: off). + let (compute_left, render_left) = self.stamps_left(); + writeln!( + f, + "stamps: compute_left {} compute_margin {} render_left {} render_margin {}", + compute_left, + params::compute_reseed_margin(), + render_left, + params::stamp_reseed_margin() + )?; writeln!( f, "stage: ta_us {} fs_us {} cdm_us {} span_us {}", From 9f2b5e61bf0bebcf1fe391fb6a9204b592e1bfb5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:46:52 -0700 Subject: [PATCH 43/46] tools/asahi: g15-display-order-tests: model the system-sleep hold The suite builds admit() from g15/boot.rs against host models. With omacom's T6031 system-sleep hold, admit() reads the ASC control word before it takes the hold, passes whether to hold through system sleep to glue::hold_power(dev, sleep_hold), reads params::sleep_hold() and warns with dev_warn! when g15_sleep_hold=0 on T6031. The model has no dev_warn!, no params module and a one-argument hold_power(), so the admit() control no longer builds. Add dev_warn! (counted), params::sleep_hold() and a failing ASC read to the model, and pick the hold_power() mock from the admit() it is given: with the sleep-hold argument it records the value passed; without it, the model is the old one. Every existing scenario and mutant runs unchanged on both. For an admit() with the sleep-hold argument, also check that the hold is taken through system sleep only on T6031 with g15_sleep_hold on, that g15_sleep_hold=0 on T6031 warns once, that T6030 never holds through sleep, and that a failing ASC read returns before any hold is taken on either SoC. Four mutants (the hold on any platform, ignoring the parameter, no warning, the ASC read after the hold) each fail one of those checks. Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 762142518c411ead6589e46259760633a2222ae1) [ryan: drop the omacom pull request number; that sleep hold is not in this tree, and the model still builds the admit() without it] Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/asahi/g15-display-order-tests.py | 81 ++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 12 deletions(-) diff --git a/tools/asahi/g15-display-order-tests.py b/tools/asahi/g15-display-order-tests.py index 50e0aa4809396..a6666efdf2776 100644 --- a/tools/asahi/g15-display-order-tests.py +++ b/tools/asahi/g15-display-order-tests.py @@ -227,7 +227,7 @@ def c_source(root): } RS_PREFIX = r''' -#![allow(dead_code, unused_variables, unused_imports, non_snake_case)] +#![allow(dead_code, unused_variables, unused_imports, unused_macros, non_snake_case)] use std::cell::RefCell; use std::marker::PhantomData; #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -236,13 +236,21 @@ def c_source(root): type Result = std::result::Result; const ENODEV: Error = Error(-19); const EINVAL: Error = Error(-22); +const EIO: Error = Error(-5); const EPROBE_DEFER: Error = Error(-517); macro_rules! dev_err { ($($args:tt)*) => {{}}; } macro_rules! dev_info { ($($args:tt)*) => {{}}; } -struct State { board: bool, machine: bool, pmp: i32, display: i32 } +macro_rules! dev_warn { ($($args:tt)*) => {{ warned(); }}; } +struct State { board: bool, machine: bool, pmp: i32, display: i32, sleep_hold: bool, asc_fail: bool } thread_local! { static CALLS: RefCell> = const { RefCell::new(Vec::new()) }; - static STATE: RefCell = const { RefCell::new(State { board: true, machine: true, pmp: 0, display: 0 }) }; + static STATE: RefCell = const { RefCell::new(State { board: true, machine: true, pmp: 0, display: 0, sleep_hold: true, asc_fail: false }) }; + static HOLD: RefCell> = const { RefCell::new(None) }; + static WARNINGS: RefCell = const { RefCell::new(0) }; +} +fn warned() { WARNINGS.with(|w| *w.borrow_mut() += 1); } +mod params { + pub fn sleep_hold() -> bool { super::STATE.with(|s| s.borrow().sleep_hold) } } fn call(name: &'static str) { CALLS.with(|c| c.borrow_mut().push(name)); } mod device { @@ -276,7 +284,9 @@ def c_source(root): pub struct Regs; impl Regs { pub fn new(_: &super::platform::Device) -> super::Result { super::call("regs"); Ok(Self) } - pub fn asc_control(&self) -> super::Result { Ok(0) } + pub fn asc_control(&self) -> super::Result { + if super::STATE.with(|s| s.borrow().asc_fail) { Err(super::EIO) } else { Ok(0) } + } } } mod glue { @@ -286,7 +296,7 @@ def c_source(root): pub unsafe fn asahi_g15_pmp_prepare(_: *mut core::ffi::c_void) -> core::ffi::c_int { call("pmp"); STATE.with(|s| s.borrow().pmp) } pub unsafe fn asahi_g15_display_prepare(_: *mut core::ffi::c_void) -> core::ffi::c_int { call("display"); STATE.with(|s| s.borrow().display) } pub fn dma_coherent(_: &super::device::Device) -> bool { call("coherent"); false } - pub fn hold_power(_: &super::device::Device) { call("hold"); } +@HOLD_POWER@ } struct Admitted { regs: regs::Regs, adt: adt::GpuAdt, board_images: u32 } ''' @@ -297,12 +307,15 @@ def c_source(root): const TAIL: [&str; 6] = ["adt", "identity", "coherent", "board_images", "regs", "hold"]; fn run(cfg: &'static Config, machine: bool, pmp: i32, display: i32) -> (Result<()>, Vec<&'static str>) { CALLS.with(|c| c.borrow_mut().clear()); - STATE.with(|s| *s.borrow_mut() = State { board: true, machine, pmp, display }); + STATE.with(|s| *s.borrow_mut() = State { board: true, machine, pmp, display, sleep_hold: true, asc_fail: false }); + HOLD.with(|h| *h.borrow_mut() = None); + WARNINGS.with(|w| *w.borrow_mut() = 0); let pdev = platform::Device(device::Device, Core(PhantomData)); let ret = admit(&pdev, &AsahiDevice, cfg).map(|_| ()); (ret, CALLS.with(|c| c.borrow().clone())) } fn with_tail(head: &[&'static str]) -> Vec<&'static str> { head.iter().chain(TAIL.iter()).copied().collect() } +@SLEEP_HOLD_RUN@ fn main() { assert_eq!(run(&T6031, true, 0, -517), (Err(EPROBE_DEFER), vec!["machine", "display"]), "T6031 defers before firmware"); assert_eq!(run(&T6031, true, 0, -19), (Err(ENODEV), vec!["machine", "display"]), "T6031 refusal stops admission"); @@ -310,9 +323,42 @@ def c_source(root): assert_eq!(run(&T6031, false, 0, 0), (Err(ENODEV), vec!["machine"]), "the machine check comes first"); assert_eq!(run(&T6030, true, 0, -517), (Ok(()), with_tail(&["board", "pmp"])), "T6030 path unchanged"); assert_eq!(run(&T6030, true, -517, 0), (Err(EPROBE_DEFER), vec!["board", "pmp"]), "T6030 still defers on its PMP link"); +@SLEEP_HOLD_CHECKS@ } ''' +HOLD_PLAIN = ' pub fn hold_power(_: &super::device::Device) { call("hold"); }' +HOLD_SLEEP = (' pub fn hold_power(_: &super::device::Device, sleep_hold: bool) {\n' + ' call("hold"); super::HOLD.with(|h| *h.borrow_mut() = Some(sleep_hold));\n }') +SLEEP_HOLD_RUN = r''' +fn run_hold(cfg: &'static Config, sleep_hold: bool, asc_fail: bool) -> (Result<()>, Vec<&'static str>, Option, u32) { + CALLS.with(|c| c.borrow_mut().clear()); + STATE.with(|s| *s.borrow_mut() = State { board: true, machine: true, pmp: 0, display: 0, sleep_hold, asc_fail }); + HOLD.with(|h| *h.borrow_mut() = None); + WARNINGS.with(|w| *w.borrow_mut() = 0); + let pdev = platform::Device(device::Device, Core(PhantomData)); + let ret = admit(&pdev, &AsahiDevice, cfg).map(|_| ()); + (ret, CALLS.with(|c| c.borrow().clone()), HOLD.with(|h| *h.borrow()), WARNINGS.with(|w| *w.borrow())) +} +fn before_hold(head: &[&'static str]) -> Vec<&'static str> { head.iter().chain(TAIL[..TAIL.len() - 1].iter()).copied().collect() } +''' +SLEEP_HOLD_CHECKS = r''' + // The system-sleep hold: T6031 only, and only while g15_sleep_hold is on; off is warned about. + assert_eq!(run_hold(&T6031, true, false), (Ok(()), with_tail(&["machine", "display"]), Some(true), 0), "T6031 holds gfx through sleep"); + assert_eq!(run_hold(&T6031, false, false), (Ok(()), with_tail(&["machine", "display"]), Some(false), 1), "g15_sleep_hold=0: no sleep hold, one warning"); + assert_eq!(run_hold(&T6030, true, false), (Ok(()), with_tail(&["board", "pmp"]), Some(false), 0), "T6030 never holds through sleep"); + // The ASC read comes before the hold: a failure there leaves no hold behind. + assert_eq!(run_hold(&T6031, true, true), (Err(EIO), before_hold(&["machine", "display"]), None, 0), "T6031: ASC failure takes no hold"); + assert_eq!(run_hold(&T6030, true, true), (Err(EIO), before_hold(&["board", "pmp"]), None, 0), "T6030: ASC failure takes no hold"); +''' +SLEEP_HOLD_MUTANTS = { + "sleep-hold-any-platform": ("let sleep_hold = t6031 && params::sleep_hold();", "let sleep_hold = params::sleep_hold();"), + "sleep-hold-ignores-parameter": ("let sleep_hold = t6031 && params::sleep_hold();", "let sleep_hold = t6031;"), + "no-warning-when-off": ("} else if t6031 {", "} else if false {"), + "asc-read-after-hold": (re.compile(r"(\n\s*)let asc = regs\.asc_control\(\)\?;(.*?glue::hold_power\(dev, sleep_hold\);)", re.S), + r"\2\1let asc = regs.asc_control()?;"), +} + RS_MUTANTS = { "display-refusal-ignored": ("dev_err!(dev, \"native display not admitted: {:?}\\n\", err);\n }\n return Err(err);", "dev_err!(dev, \"native display not admitted: {:?}\\n\", err);\n }"), @@ -346,10 +392,18 @@ def c_failures(exe): return [s for s in C_SCENARIOS if subprocess.run([str(exe), s], capture_output=True).returncode] -def compile_rs(work, rustc, source, name): +def rs_model(admit): + """The prelude and main for this admit(): with the system-sleep hold argument (g15_sleep_hold) or without.""" + sleep = "glue::hold_power(dev, sleep_hold);" in admit + prefix = RS_PREFIX.replace("@HOLD_POWER@", HOLD_SLEEP if sleep else HOLD_PLAIN) + main = RS_MAIN.replace("@SLEEP_HOLD_RUN@", SLEEP_HOLD_RUN if sleep else "").replace("@SLEEP_HOLD_CHECKS@", SLEEP_HOLD_CHECKS if sleep else "") + return sleep, prefix, main + + +def compile_rs(work, rustc, source, name, model): src = work / f"{name}.rs" exe = work / name - src.write_text(RS_PREFIX + source + RS_MAIN) + src.write_text(model[1] + source + model[2]) result = subprocess.run([rustc, "--edition=2021", "-o", str(exe), str(src)], capture_output=True, text=True) if result.returncode: raise SystemExit(f"{name}: does not build with {rustc}:\n{result.stderr}") @@ -387,6 +441,8 @@ def main(): checks = static_checks(args.root) source = c_source(args.root) admit = item((args.root / BOOT).read_text(), "pub(crate) fn admit(") + model = rs_model(admit) + rs_mutants = dict(RS_MUTANTS, **(SLEEP_HOLD_MUTANTS if model[0] else {})) with tempfile.TemporaryDirectory(prefix="g15-display-order-", dir=os.environ.get("TMPDIR")) as tmp: work = Path(tmp) failed = c_failures(compile_c(work, source, "control")) @@ -396,16 +452,17 @@ def main(): assert failed, f"C mutant {name} survived" print(f"mutant {name}: caught by {', '.join(failed)}") for rustc in rustcs: - exe = compile_rs(work, rustc, admit, "admit") + exe = compile_rs(work, rustc, admit, "admit", model) run = subprocess.run([str(exe)], capture_output=True, text=True) assert run.returncode == 0, f"admit() control failed with {rustc}:\n{run.stderr}" - for name, change in RS_MUTANTS.items(): - exe = compile_rs(work, rustc, mutate(admit, change), f"admit-{name}") + for name, change in rs_mutants.items(): + exe = compile_rs(work, rustc, mutate(admit, change), f"admit-{name}", model) run = subprocess.run([str(exe)], capture_output=True, text=True) assert run.returncode, f"admit() mutant {name} survived with {rustc}" print(f"mutant {name} ({rustc}): caught") print(f"PASS: {checks} static checks, {len(C_SCENARIOS)} C scenarios, " - f"{len(C_MUTANTS)} C mutants, admit() on {len(rustcs)} rustc, {len(RS_MUTANTS)} admit() mutants") + f"{len(C_MUTANTS)} C mutants, admit() on {len(rustcs)} rustc, {len(rs_mutants)} admit() mutants" + + (", system-sleep hold checks" if model[0] else "")) if __name__ == "__main__": From c0cd9aa5889c75406ca95e3639e54e8ad0688c63 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:29:09 +1100 Subject: [PATCH 44/46] drm/asahi: g15: m3: make the PB start and idle reclaim fixed driver values The parameter buffer series came with four new module parameters: g15_pb_blocks (initial posting, 64), g15_pb_reclaim (2), g15_pb_idle_ms (1000) and g15_pb_reclaim_blocks (1024). The fleet kernel builds the aurora module with G15 off, and the module! parameters are not gated on it, so all four would show up in /sys/module/aurora/parameters on every Mac, and on an M3 they are knobs nobody should need to touch. Make them constants in hw::tuning with the same values: PB_BLOCKS, PB_RECLAIM, PB_IDLE_MS and PB_RECLAIM_BLOCKS. params::pb_blocks(), pb_reclaim(), pb_idle_ns() and pb_reclaim_blocks() now return them, so the PB code is unchanged. The behaviour on an M3 is the series default. g15-pb-growth-tests.py checks that production's four accessors return exactly those constants, and its own accessors start from them, so its cases that vary the values still run. The zero-words harness pins the new hw.rs bytes; the T6030 tables are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/asahi_core.rs | 16 ------ drivers/gpu/drm/asahi/g15/hw.rs | 11 +++- drivers/gpu/drm/asahi/g15/params.rs | 13 +++-- drivers/gpu/drm/asahi/g15/pools.rs | 20 +++---- drivers/gpu/drm/asahi/g15/pools/growth.rs | 6 +-- tools/asahi/g15-pb-growth-tests.py | 66 +++++++++++------------ tools/asahi/g15-zero-words-tests.py | 7 ++- 7 files changed, 67 insertions(+), 72 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index d5daef485f06a..0e70187c33bf4 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -173,22 +173,6 @@ module! { default: 3072, description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", }, - g15_pb_blocks: u32 { - default: 64, - description: "G15: PB blocks posted at (re)initialization, 128 KiB each (64..4095)", - }, - g15_pb_reclaim: u32 { - default: 2, - description: "G15: PB reclaim when idle: 0 never, 1 re-initialize, 2 also free backing", - }, - g15_pb_idle_ms: u32 { - default: 1000, - description: "G15: idle time before a PB reclaim, ms (at least 100)", - }, - g15_pb_reclaim_blocks: u32 { - default: 1024, - description: "G15: PB blocks posted above which an idle engine reclaims the PB", - }, g15_stamp_base: u32 { default: 0, description: "G15 (testing): render ordinal the TA/FS stamps start from; 0 = unchanged", diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index 3768907f7416e..cf868492edfa3 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -581,8 +581,17 @@ pub(crate) mod tuning { pub(crate) const SLOT_ROTATE: bool = true; /// Flights per VM. pub(crate) const MAX_INFLIGHT: u32 = 2; - /// PB growth through event 7 / DEVCTL 7; the initial posting is `g15_pb_blocks` (64). + /// PB growth through event 7 / DEVCTL 7, from the initial posting of [`PB_BLOCKS`]. pub(crate) const PB_GROW: bool = true; + /// PB blocks posted at (re)initialization: 64 blocks of 128 KiB (8 MiB). + pub(crate) const PB_BLOCKS: u32 = 64; + /// PB reclaim when the engine is idle: 0 never, 1 re-initialize, 2 also free the backing + /// above the initial posting and the reserve. + pub(crate) const PB_RECLAIM: u32 = 2; + /// Idle time before a PB reclaim, ms. + pub(crate) const PB_IDLE_MS: u32 = 1000; + /// Posted PB blocks above which an idle engine reclaims the PB. + pub(crate) const PB_RECLAIM_BLOCKS: u32 = 1024; /// The most PB blocks that may be posted: one below the mapped heap's block count, so the /// block-list producer (modulo K = 4096) never returns to 0. pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS - 1; diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index a5549c3010a6f..1f5d39c06ae27 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -111,25 +111,24 @@ pub(crate) fn pb_max_blocks() -> u32 { module_parameters::g15_pb_max_blocks.value().clamp(64, super::hw::tuning::PB_MAX_BLOCKS) } -/// The initial PB posting, in blocks (64..=4095; the ledger clamps it again). Read-only after -/// boot. +/// The initial PB posting, in blocks: a fixed driver value, not a module parameter. pub(crate) fn pb_blocks() -> u32 { - module_parameters::g15_pb_blocks.value().clamp(64, super::hw::tuning::PB_MAX_BLOCKS) + super::hw::tuning::PB_BLOCKS } /// PB reclaim at idle: 0 never, 1 re-initialize, 2 re-initialize and free the backing. pub(crate) fn pb_reclaim() -> u32 { - module_parameters::g15_pb_reclaim.value().min(2) + super::hw::tuning::PB_RECLAIM } -/// Idle time before a PB reclaim, ns (at least 100 ms). +/// Idle time before a PB reclaim, ns. pub(crate) fn pb_idle_ns() -> u64 { - module_parameters::g15_pb_idle_ms.value().max(100) as u64 * 1_000_000 + super::hw::tuning::PB_IDLE_MS as u64 * 1_000_000 } /// Posted blocks above which an idle engine reclaims the PB. pub(crate) fn pb_reclaim_blocks() -> u32 { - module_parameters::g15_pb_reclaim_blocks.value() + super::hw::tuning::PB_RECLAIM_BLOCKS } /// Whether FRAGMENT_BARRIERS is advertised and mapped to early tiling. diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index 8a3989f723abb..668d7b408dcc4 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -12,7 +12,7 @@ //! the fixed client GVA: each VM's before its prefixes are published ([`Firmware::map_pb`]), //! and a chunk backed later also into the VM of the flights; nothing names its pages through //! the host window or context 0 (the records hold client GVAs only). Initial -//! `g15_pb_blocks` (64), at most [`super::hw::tuning::PB_MAX_BLOCKS`] (4095) posted, so the +//! `tuning::PB_BLOCKS` (64), at most [`super::hw::tuning::PB_MAX_BLOCKS`] (4095) posted, so the //! producer never returns to 0; the firmware is given [`Pools::soft_max`] (default 3072) as //! its growth maximum. Truthful K = 4096 in every //! profile (PB reconstruction, also in recovery, synchronizes 8 * K bytes of the block list: @@ -21,15 +21,15 @@ //! replies of a pass, while fewer than a reserve chunk remain, [`Firmware::refill_backing`] //! backs the next chunk, so a reply never waits for the allocator. Only a reserve that ran dry //! makes a request wait: it stays queued, the pass backs a chunk and the next pass answers it. -//! - Reclaim ([`Firmware::reclaim_check`], `g15_pb_reclaim`): posted blocks never come back +//! - Reclaim ([`Firmware::reclaim_check`], `tuning::PB_RECLAIM`): posted blocks never come back //! while a PB lives (the firmware recycles pages inside its own pool; the host has no view of -//! it), so an engine idle for `g15_pb_idle_ms` with more than `g15_pb_reclaim_blocks` posted +//! it), so an engine idle for `tuning::PB_IDLE_MS` with more than `tuning::PB_RECLAIM_BLOCKS` posted //! re-initializes the PB at its next render through the re-initialization path of a recovered //! abort (fresh lists and manager, InitBM of the initial blocks, no halt) and frees the backing //! above the kept chunks (also with fewer posted, when more is backed than they need); an //! engine idle for 50 ms (a client boundary) with more than half the maximum posted //! re-initializes it the same way but keeps the backing; if no render follows, with -//! `g15_pb_reclaim` 2 the backing above the kept chunks is freed after `g15_pb_idle_ms`. +//! `tuning::PB_RECLAIM` 2 the backing above the kept chunks is freed after `tuning::PB_IDLE_MS`. //! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each //! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, //! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) @@ -82,7 +82,7 @@ static_assert!(buffer::PAGE_LIST_SIZE as u64 == client::PB_PAGE_LIST_SIZE); static_assert!(uma_manager::SEEDS == sizes::FREELIST_SEED); static_assert!(uma_pools::POOL_SIZE == sizes::POOL_SIZE); // The ledger keeps the driver's PB maximum: it is not capped by [`growth::MAX_POSTED`]. The -// initial posting (`g15_pb_blocks`) is clamped to MIN_BLOCKS..=PB_MAX_BLOCKS. +// initial posting (`tuning::PB_BLOCKS`) is clamped to MIN_BLOCKS..=PB_MAX_BLOCKS. static_assert!(tuning::PB_MAX_BLOCKS <= growth::MAX_POSTED); static_assert!(buffer::MIN_BLOCKS <= tuning::PB_MAX_BLOCKS); // The PB alias is GPU-only (not cached and firmware-visible): dropping one is a plain unmap, @@ -700,17 +700,17 @@ impl Firmware { self.pools.idle.note(idle, super::engine::now_ns()); } - /// Reclaims the PB when the engine has been idle for `g15_pb_idle_ms` with more than - /// `g15_pb_reclaim_blocks` posted or more backing than the posted blocks need, or for + /// Reclaims the PB when the engine has been idle for `tuning::PB_IDLE_MS` with more than + /// `tuning::PB_RECLAIM_BLOCKS` posted or more backing than the posted blocks need, or for /// [`growth::DRAIN_IDLE_NS`] with more than half the maximum posted /// ([`growth::Ledger::drain_reinit_due`]; re-initialization only): the next render prefix /// re-initializes the render allocators and the PB (InitBM of the initial blocks; /// `recovery.reinit`, as after a recovered abort but with no halt), and with - /// `g15_pb_reclaim` 2 the backing chunks above those of the initial posting and the reserve + /// `tuning::PB_RECLAIM` 2 the backing chunks above those of the initial posting and the reserve /// are freed now, unmapped from every VM of the registry (a VM that publishes later drops /// its stale mappings at [`VmState::map_heap`]). While a reclaim's re-initialization is - /// pending (a client-boundary one kept the backing), `g15_pb_reclaim` 2 frees the backing - /// above those chunks once the engine has been idle for `g15_pb_idle_ms` + /// pending (a client-boundary one kept the backing), `tuning::PB_RECLAIM` 2 frees the backing + /// above those chunks once the engine has been idle for `tuning::PB_IDLE_MS` /// ([`growth::Ledger::pending_free_due`]). Called under the device lock when a job arrives /// at an idle engine and in idle poll passes. pub(crate) fn reclaim_check(&mut self) { diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs index fef40a6d3a73e..e9eaee242ac51 100644 --- a/drivers/gpu/drm/asahi/g15/pools/growth.rs +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -197,7 +197,7 @@ impl Ledger { /// Whether a short idle re-initializes the heap: more than half the maximum posted and /// idle for [`DRAIN_IDLE_NS`]. Back-to-back heavy scenes (a new client every few hundred - /// ms) never reach `g15_pb_idle_ms`, and posted blocks never come back while a PB lives, so + /// ms) never reach `tuning::PB_IDLE_MS`, and posted blocks never come back while a PB lives, so /// without this they run the heap to the maximum. A re-initialization with no idle time /// faulted at the heap base (at a client switch, the second in a row with no recovery /// between); with 50 ms of idle none did, also between two renders of one client. @@ -381,7 +381,7 @@ mod tests { assert_eq!(Ledger::new(true, 512, 100).backed, 0); let fixed = Ledger::new(false, 512, 1024); assert_eq!((fixed.initial, fixed.max, fixed.posted), (4095, 4095, 4095)); - // The values the driver passes (g15_pb_blocks 64, tuning::PB_MAX_BLOCKS) are kept. + // The values the driver passes (tuning::PB_BLOCKS 64, tuning::PB_MAX_BLOCKS) are kept. let l = Ledger::new(true, 64, 4095); assert_eq!( (l.initial, l.max, l.posted, l.producer()), @@ -814,7 +814,7 @@ mod tests { } } // The case of the J516C run: a client-boundary re-initialization left 23 chunks for - // 264 posted blocks; the free is due after g15_pb_idle_ms of idle. + // 264 posted blocks; the free is due after tuning::PB_IDLE_MS of idle. let mut l = Ledger::new(true, 64, 4095); l.unback_to(23); l.posted = 264; diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index cbe1280141244..a1f2d67d7a327 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -6,11 +6,11 @@ answer_request(), map_pb(), refill_backing(), note_idle(), reclaim_check(), free_backing() and reclaim_delay_ns(), Pools::soft_max(), map_heap(), free_chunks() and back_chunk(), PbChunk, chunk_gva(), CHUNK_BYTES and Pools::new's PB backing statements; from vm.rs VmState::map_one(), -map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks(), pb_blocks(), pb_reclaim(), -pb_idle_ns() and pb_reclaim_blocks() from params.rs; the PB sizes, tuning and PB leaf word from -hw.rs; Prot::from_pte() and is_cached_noncoherent() from pgtable.rs; and the parameters' defaults -from asahi_core.rs. It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies -run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() +map_heap(), map_chunk() and unmap_heap_above(); pb_max_blocks() from params.rs, whose +pb_blocks(), pb_reclaim(), pb_idle_ns() and pb_reclaim_blocks() must return the fixed hw.rs tuning +values; the PB sizes, tuning and PB leaf word from hw.rs; Prot::from_pte() and +is_cached_noncoherent() from pgtable.rs; and the g15_pb_max_blocks default from asahi_core.rs. +It includes pools/growth.rs, fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, the GEM backing allocation, the shared layer's map_at() and the DEVCTL wait, which logs every allocation, alias map, unmap, free, block record, barrier, producer store, reply and log line in order. Allocation and alias map failures can be injected. Source checks: publish_prefix_into() starts with the PB catch-up, and pools.rs asserts the PB leaf @@ -451,7 +451,8 @@ def consts(text, names): pub static LOG: RefCell> = RefCell::new(Vec::new()); pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); pub static PARAM_BLOCKS: Cell = Cell::new(PB_BLOCKS_DEFAULT); - /// The reclaim parameters (g15_pb_reclaim, g15_pb_idle_ms, g15_pb_reclaim_blocks). + /// The initial posting and the reclaim values (hw::tuning PB_BLOCKS, PB_RECLAIM, + /// PB_IDLE_MS and PB_RECLAIM_BLOCKS), which a test may vary from the driver's. pub static PARAM_RECLAIM: Cell = Cell::new(PB_RECLAIM_DEFAULT); pub static PARAM_IDLE_MS: Cell = Cell::new(PB_IDLE_MS_DEFAULT); pub static PARAM_RECLAIM_BLOCKS: Cell = Cell::new(PB_RECLAIM_BLOCKS_DEFAULT); @@ -507,22 +508,6 @@ def consts(text, names): } } pub static g15_pb_max_blocks: Param = Param; - pub struct Blocks; - impl Blocks { - pub fn value(&self) -> u32 { - crate::PARAM_BLOCKS.with(|p| p.get()) - } - } - pub static g15_pb_blocks: Blocks = Blocks; - pub struct Cellp(pub &'static std::thread::LocalKey>); - impl Cellp { - pub fn value(&self) -> u32 { - self.0.with(|p| p.get()) - } - } - pub static g15_pb_reclaim: Cellp = Cellp(&crate::PARAM_RECLAIM); - pub static g15_pb_idle_ms: Cellp = Cellp(&crate::PARAM_IDLE_MS); - pub static g15_pb_reclaim_blocks: Cellp = Cellp(&crate::PARAM_RECLAIM_BLOCKS); } pub mod driver { @@ -1832,7 +1817,7 @@ def consts(text, names): assert!(take().is_empty()); } - /// With g15_pb_reclaim 1 the client-boundary re-initialization keeps the backing for good. + /// With a reclaim level of 1 (tuning::PB_RECLAIM is 2) the client-boundary re-initialization keeps the backing for good. #[test] fn client_boundary_reinit_at_level_1_keeps_the_backing() { defaults(); @@ -1852,7 +1837,7 @@ def consts(text, names): } /// Nothing grown since the initialization: an initial posting above half the maximum - /// (g15_pb_blocks 3000), or a floor below it (g15_pb_reclaim_blocks 32): no + /// (an initial posting of 3000), or a floor below it (a reclaim floor of 32): no /// re-initialization at any idle. #[test] fn nothing_grown_reinitializes_nothing() { @@ -2121,23 +2106,38 @@ def run(mutation): if init.count(line) != 1: raise SystemExit('Pools::init() does not hold: ' + line) reset = '\n'.join('p.' + line[len('self.'):] for line in LEDGER_RESET) - pb_max_blocks = '\n'.join(item(src[PARAMS], f'pub(crate) fn {name}() -> ') - for name in ('pb_max_blocks', 'pb_blocks', 'pb_reclaim', 'pb_idle_ns', - 'pb_reclaim_blocks')) + pb_max_blocks = item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> ') + # The initial posting and the reclaim values are fixed driver values (hw::tuning), not + # module parameters: production's accessors must return exactly those constants. The + # harness's accessors read cells that start at them, so a test can vary them. + fixed = {'pb_blocks': ('u32', 'super::hw::tuning::PB_BLOCKS', 'PARAM_BLOCKS'), + 'pb_reclaim': ('u32', 'super::hw::tuning::PB_RECLAIM', 'PARAM_RECLAIM'), + 'pb_idle_ns': ('u64', 'super::hw::tuning::PB_IDLE_MS as u64 * 1_000_000', 'PARAM_IDLE_MS'), + 'pb_reclaim_blocks': ('u32', 'super::hw::tuning::PB_RECLAIM_BLOCKS', 'PARAM_RECLAIM_BLOCKS')} + for name, (ty, body, cell) in fixed.items(): + got = item(src[PARAMS], f'pub(crate) fn {name}() -> ') + want = f'pub(crate) fn {name}() -> {ty} {{\n {body}\n}}' + if got.strip() != want: + raise SystemExit(f'params.rs {name}() is not the fixed tuning value: {got!r}') + value = f'crate::{cell}.with(|p| p.get())' + if name == 'pb_idle_ns': + value = f'{value} as u64 * 1_000_000' + pb_max_blocks += f'\npub(crate) fn {name}() -> {ty} {{\n {value}\n}}\n' + tuning_hw = item(hw, 'pub(crate) mod tuning {') reclaim_defaults = {} - for param, const in (('g15_pb_reclaim', 'PB_RECLAIM_DEFAULT'), ('g15_pb_idle_ms', 'PB_IDLE_MS_DEFAULT'), - ('g15_pb_reclaim_blocks', 'PB_RECLAIM_BLOCKS_DEFAULT')): - m = re.search(rf'{param}: u32 \{{\s*default: (\d+),', src[CORE]) + for name, const in (('PB_RECLAIM', 'PB_RECLAIM_DEFAULT'), ('PB_IDLE_MS', 'PB_IDLE_MS_DEFAULT'), + ('PB_RECLAIM_BLOCKS', 'PB_RECLAIM_BLOCKS_DEFAULT')): + m = re.search(rf'pub\(crate\) const {name}: u32 = (\d+);', tuning_hw) if not m: - raise SystemExit(f'{param} default not found') + raise SystemExit(f'tuning::{name} not found') reclaim_defaults[const] = int(m.group(1)) m = re.search(r'g15_pb_max_blocks: u32 \{\s*default: (\d+),', src[CORE]) if not m: raise SystemExit('g15_pb_max_blocks default not found') param_default = int(m.group(1)) - m = re.search(r'g15_pb_blocks: u32 \{\s*default: (\d+),', src[CORE]) + m = re.search(r'pub\(crate\) const PB_BLOCKS: u32 = (\d+);', tuning_hw) if not m: - raise SystemExit('g15_pb_blocks default not found') + raise SystemExit('tuning::PB_BLOCKS not found') blocks_default = int(m.group(1)) sizes = consts(item(hw, 'pub(crate) mod sizes {'), ['PB_BLOCKS']) tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_MAX_BLOCKS']) diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index 886b0ab3be2c6..a2dfa5891cd91 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -12,7 +12,10 @@ the T6030 tables are unchanged), and for publication-bound slots (removal of MAX_VMS and its lifetime-slot comment only), and when Board::DeviceTree was gated on CONFIG_DRM_ASAHI_G15 for the T8122 runtime without the G15 backend -(one #[cfg] line; the tables are unchanged). Its T6030 configuration module is +(one #[cfg] line; the tables are unchanged), and when the initial PB posting +and the idle reclaim became fixed tuning constants instead of module +parameters (four constants; the tables are unchanged). Its T6030 +configuration module is independently pinned to the original bytes, except for the J514S B0 FirmwareBuild row added to its firmware table: the module is the original pinned bytes (0079ae08) plus that one 15-line row, and @@ -33,7 +36,7 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "9bd94ec8654736e1295061925f3c47264be8d338ad4dcb38b8d66b3dfcef2fa7", + "hw.rs": "4480fed5406a8c89f53aa4d63c5217705d68265eb4f06f00868460d535ff69a9", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { From a91ef825ace8f475abda01a4ea57f413a91130c1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:32:12 +1100 Subject: [PATCH 45/46] drm/asahi: g15: renumber the stamps at a fixed margin, without module parameters The stamp renumbering series added g15_stamp_reseed_margin and g15_compute_reseed_margin, both 0 (renumbering off) by default, and two testing parameters that start the stamps at a chosen ordinal. Off by default, the M3 GPU still reaches the TA stamp wrap after hours of desktop use, and the wrap through zero has not been tried on hardware. The parameters would also show up on every fleet kernel, G15 or not. Renumber at a fixed margin of 100000 renders and 100000 computes before each wrap, the margin the series was run with on a J516C, as hw::tuning::STAMP_RESEED_MARGIN and COMPUTE_RESEED_MARGIN. Compile-time checks keep them inside the bounds the parameters were clamped to. Drop the two testing parameters: the stamps start from their seeds again, as before the series, and the engine's stamp bases start at 0. The stats line prints the fixed margins. The zero-words harness pins the new hw.rs bytes; the T6030 tables are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/asahi_core.rs | 16 ---------- drivers/gpu/drm/asahi/g15/engine.rs | 4 +-- drivers/gpu/drm/asahi/g15/graph.rs | 8 ++--- drivers/gpu/drm/asahi/g15/hw.rs | 6 ++++ drivers/gpu/drm/asahi/g15/params.rs | 46 ++++++++++------------------- drivers/gpu/drm/asahi/g15/poll.rs | 2 +- tools/asahi/g15-zero-words-tests.py | 5 ++-- 7 files changed, 30 insertions(+), 57 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 0e70187c33bf4..decf16ea09647 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -173,22 +173,6 @@ module! { default: 3072, description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", }, - g15_stamp_base: u32 { - default: 0, - description: "G15 (testing): render ordinal the TA/FS stamps start from; 0 = unchanged", - }, - g15_stamp_reseed_margin: u32 { - default: 0, - description: "G15: renders before the TA stamp wrap within which TA/FS are renumbered while idle; 0 = off", - }, - g15_compute_stamp_base: u32 { - default: 0, - description: "G15 (testing): compute ordinal the compute stamp starts from; 0 = unchanged", - }, - g15_compute_reseed_margin: u32 { - default: 0, - description: "G15: computes before the compute stamp wrap within which it is renumbered while idle; 0 = off", - }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 1a00f33e0c96b..1fa5c60bd1122 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -417,8 +417,8 @@ impl Engine { shared: None, ordinal: 0, draw: 0, - stamp_base: params::stamp_base(), - compute_stamp_base: params::compute_stamp_base(), + stamp_base: 0, + compute_stamp_base: 0, target: 0, head: 0, ta_target: 0, diff --git a/drivers/gpu/drm/asahi/g15/graph.rs b/drivers/gpu/drm/asahi/g15/graph.rs index 18680fd05596f..9a62f25f19a63 100644 --- a/drivers/gpu/drm/asahi/g15/graph.rs +++ b/drivers/gpu/drm/asahi/g15/graph.rs @@ -251,8 +251,7 @@ impl ComputeGraph { q.w64(qinfo::UNK_AC, 4)?; self.qstate.w32(qstate::CAP, sizes::QUEUE_CAP)?; - let start = super::encode::stamp(super::params::compute_stamp_base()); - self.user_stamp.w32(0, start)?; + self.user_stamp.w32(0, engine::COMPUTE_SEED)?; self.fw_stamp.w32(0, 0) } } @@ -353,9 +352,8 @@ impl RenderGraph { q.w64(qinfo::UNK_AC, 4)?; self.qstate[e].w32(qstate::CAP, sizes::QUEUE_CAP)?; self.ring[e].fill(0, 0xcc, sizes::QUEUE_CAP as usize * 8)?; - let start = super::encode::stamp_seed(seed[e], super::params::stamp_base()); - self.user_stamp[e].w32(0, start)?; - self.fw_stamp[e].w32(0, start)?; + self.user_stamp[e].w32(0, seed[e])?; + self.fw_stamp[e].w32(0, seed[e])?; } init_sched_ctx(&self.sched_ctx)?; init_notifier(&self.notifier, &self.threshold)?; diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index cf868492edfa3..b211088310210 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -592,6 +592,12 @@ pub(crate) mod tuning { pub(crate) const PB_IDLE_MS: u32 = 1000; /// Posted PB blocks above which an idle engine reclaims the PB. pub(crate) const PB_RECLAIM_BLOCKS: u32 = 1024; + /// Renders before the TA stamp wrap within which the TA/FS stamps are renumbered while + /// nothing is in flight (the wrap is at about 16.7 million renders). + pub(crate) const STAMP_RESEED_MARGIN: u64 = 100_000; + /// Computes before the compute stamp wrap within which the compute stamp is renumbered + /// while nothing is in flight (the wrap is at 4128768 computes). + pub(crate) const COMPUTE_RESEED_MARGIN: u64 = 100_000; /// The most PB blocks that may be posted: one below the mapped heap's block count, so the /// block-list producer (modulo K = 4096) never returns to 0. pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS - 1; diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs index 1f5d39c06ae27..c2c3486475061 100644 --- a/drivers/gpu/drm/asahi/g15/params.rs +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -18,44 +18,28 @@ use crate::module_parameters; /// updates); this bounds the kernel's staging allocation. pub(crate) const MAX_BINDS: u32 = 1 << 20; -/// Render ordinal the TA/FS stamps start from. Only for testing the stamp wrap without hours of -/// drawing; 0 keeps the stamps exactly as before. -pub(crate) fn stamp_base() -> u64 { - (module_parameters::g15_stamp_base.value() as u64) % super::encode::STAMP_PERIOD -} +// At least one prefix of renders, so that the last prefix published outside the margin cannot +// cross the wrap; at most half the stamp range. +static_assert!( + hw::tuning::STAMP_RESEED_MARGIN >= hw::tuning::PREFIX_RENDERS as u64 + && hw::tuning::STAMP_RESEED_MARGIN <= super::encode::RENDER_WRAP / 2 +); +// At least one prefix of computes; at most half the stamp range. +static_assert!( + hw::tuning::COMPUTE_RESEED_MARGIN >= hw::sizes::COMPUTE_SLOTS as u64 + && hw::tuning::COMPUTE_RESEED_MARGIN <= super::encode::COMPUTE_WRAP / 2 +); /// Renders before the TA stamp wrap within which the TA/FS stamps are renumbered from their seeds, -/// at the first moment with nothing in flight (0: never). At least one prefix of renders, so -/// that the last prefix published outside the margin cannot cross the wrap; at most half the -/// stamp range. +/// at the first moment with nothing in flight ([`hw::tuning::STAMP_RESEED_MARGIN`]). pub(crate) fn stamp_reseed_margin() -> u64 { - match module_parameters::g15_stamp_reseed_margin.value() as u64 { - 0 => 0, - m => m.clamp( - hw::tuning::PREFIX_RENDERS as u64, - super::encode::RENDER_WRAP / 2, - ), - } -} - -/// Compute ordinal the compute stamp starts from. Only for testing the stamp wrap without -/// millions of computes; 0 keeps the stamp exactly as before. -pub(crate) fn compute_stamp_base() -> u64 { - (module_parameters::g15_compute_stamp_base.value() as u64) % super::encode::STAMP_PERIOD + hw::tuning::STAMP_RESEED_MARGIN } /// Computes before the compute stamp wrap within which the compute stamp is renumbered from its -/// seed, at the first moment with nothing in flight (0: never). At least one prefix of computes, -/// so that the last prefix published outside the margin cannot cross the wrap; at most half the -/// stamp range. +/// seed, at the first moment with nothing in flight ([`hw::tuning::COMPUTE_RESEED_MARGIN`]). pub(crate) fn compute_reseed_margin() -> u64 { - match module_parameters::g15_compute_reseed_margin.value() as u64 { - 0 => 0, - m => m.clamp( - hw::sizes::COMPUTE_SLOTS as u64, - super::encode::COMPUTE_WRAP / 2, - ), - } + hw::tuning::COMPUTE_RESEED_MARGIN } /// Host deadline of a published prefix without progress, ms (at least 100). diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 282a09c4c4e11..b0356376ecf96 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -737,7 +737,7 @@ impl Firmware { s.hz_ahead )?; self.pools.write_stats(f)?; - // What is left before the stamps wrap, and the renumbering margins (0: off). + // What is left before the stamps wrap, and the renumbering margins. let (compute_left, render_left) = self.stamps_left(); writeln!( f, diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index a2dfa5891cd91..c6e32d76cb415 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -14,7 +14,8 @@ gated on CONFIG_DRM_ASAHI_G15 for the T8122 runtime without the G15 backend (one #[cfg] line; the tables are unchanged), and when the initial PB posting and the idle reclaim became fixed tuning constants instead of module -parameters (four constants; the tables are unchanged). Its T6030 +parameters (four constants; the tables are unchanged), and when the two stamp +renumbering margins became tuning constants (the tables are unchanged). Its T6030 configuration module is independently pinned to the original bytes, except for the J514S B0 FirmwareBuild row added to its firmware table: the @@ -36,7 +37,7 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "4480fed5406a8c89f53aa4d63c5217705d68265eb4f06f00868460d535ff69a9", + "hw.rs": "c54a4f8342fb466f82f4b9cfed6a137ee1ca1f4d80ede4e1b7f397d39223558e", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { From 444919ed4cb75e6c685583b7fb8984d0427b243f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 1 Nov 2026 11:39:38 +1100 Subject: [PATCH 46/46] selftests: asahi-m4: follow Uat::bind() into bind_inner() Binding G15 slots at publication split the shared Uat::bind() into bind_inner(vm, wait), with bind() as its waiting form and the G15-only try_bind() as the other. The M4 MMU retention suite extracted bind() alone and no longer built. Extract bind_inner() with it and give the slot allocator model the non-waiting try_get(). The cases are unchanged: bind() still takes the waiting path, which is the only one the G16 and G17 backends use. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/gpu/asahi-m4/test-mmu-retention.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py index 787992e9b4c24..124314ac83a5d 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py @@ -70,7 +70,7 @@ def body(source,marker): bindings=r''' #[derive(Clone,Copy)]struct Token(u32);impl Token{fn last_slot(&self)->u32{self.0}} struct Slot;impl Slot{fn changed(&self)->bool{true}fn slot(&self)->u32{3}fn token(&self)->Token{Token(3)}} -struct Allocator;impl Allocator{fn set_limit(&self,_l:Option){}fn get(&self,_t:Option)->Result{Ok(Slot)}} +struct Allocator;impl Allocator{fn set_limit(&self,_l:Option){}fn get(&self,_t:Option)->Result{Ok(Slot)}fn try_get(&self,_t:Option)->Result{Ok(Slot)}} struct VmBinding{active_users:usize,bind_token:Option,binding:Option,generation:u8,ttb:u64} #[derive(Clone)]struct Vm{id:u64,binding:Rc>} struct VmBind(Vm,u32); @@ -78,7 +78,8 @@ def body(source,marker): mod module_parameters{pub struct Param;impl Param{pub fn value(&self)->u32{0}}pub static robust_isolation:Param=Param;} ''' helpers='\n'.join(body(mm,m) for m in ['const fn binding_wire_generation(', 'const fn application_ttb1(']) -bind='impl Uat{'+body(mm,'pub(crate) fn bind(&self, vm: &Vm)')+'}\n' +# bind() is the waiting form of bind_inner(); G15's try_bind() is the other. +bind='impl Uat{'+body(mm,'pub(crate) fn bind(&self, vm: &Vm)')+'\n'+body(mm,'fn bind_inner(&self, vm: &Vm, wait: bool)')+'}\n' vm=r''' struct VmInner{id:u64,binding:Rc>,uat_inner:Rc,page_table:UatPageTable} impl VmInner{fn ttb(&self)->u64{self.page_table.ttb}}