diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs index 38f2444494805c..a58ed36cda60a4 100644 --- a/drivers/gpu/drm/asahi/g15/boot.rs +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -73,6 +73,7 @@ pub(crate) mod glue { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; pub(crate) fn asahi_g15_pmp_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_get(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + pub(crate) fn asahi_g15_pmp_ready() -> core::ffi::c_int; pub(crate) fn asahi_g15_pmp_put(dev: *mut core::ffi::c_void); fn asahi_g15_reserved_uat(index: u32, start: u64, size: u64) -> bool; } diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 45b65d1ddc026f..78b5058c8d33f7 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -146,6 +146,11 @@ pub(crate) struct Context<'a> { pub(crate) usc_base: u64, } +struct CheckedContext<'a> { + vm: &'a mut mmu::VmReadGuard, + usc_base: u64, +} + /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are /// checked ([`Prepared::validate`]). pub(crate) struct Prepared<'a> { @@ -376,7 +381,13 @@ impl Timestamps<'_> { /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a /// page without the access EACCES. -pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result { +pub(crate) fn cover( + vm: &mut mmu::VmReadGuard, + start: u64, + end: u64, + read: bool, + write: bool, +) -> Result { let mask = mmu::UAT_PGMSK as u64; if end <= start { return Err(EINVAL); @@ -406,7 +417,11 @@ pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) } /// Attachment spans, 128-byte granular, actually mapped with `write` or read access. -fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result { +fn check_attachments( + vm: &mut mmu::VmReadGuard, + list: &[uapi::drm_asahi_attachment], + write: bool, +) -> Result { for attachment in list { let end = attachment .pointer @@ -466,8 +481,8 @@ fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) { } /// Per-command render admission. -fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { - let vm = context.vm; +fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { + let vm = &mut *context.vm; let allowed = PROCESS_EMPTY_TILES | NO_VERTEX_CLUSTERING | RSRC_SPEC_HI @@ -551,11 +566,11 @@ fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Resu /// Per-command compute admission. fn check_compute( - context: &Context<'_>, + context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_compute, attachments: &[uapi::drm_asahi_attachment], ) -> Result { - let vm = context.vm; + let vm = &mut *context.vm; let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end); if cmd.flags != 0 || cmd.sampler_count != 0 @@ -649,6 +664,12 @@ impl Prepared<'_> { counts, timestamps, } = self; + // On error, unlock before the last timestamp mapping can drop and unmap. + let mut access = context.vm.read_lock()?; + let mut context = CheckedContext { + vm: &mut access, + usc_base: context.usc_base, + }; let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?; let mut compute = KVec::new(); let mut index = 0; @@ -657,12 +678,12 @@ impl Prepared<'_> { match entry.kind { SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => { // Validated readable, never serialized on G15. - check_attachments(context.vm, &attachment_list(entry.payload)?, false)?; + check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?; } SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?, CMD_RENDER => { let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; - check_render(context, &cmd)?; + check_render(&mut context, &cmd)?; commands.push( Command::Render(KBox::new( RenderCommand { @@ -677,7 +698,7 @@ impl Prepared<'_> { } CMD_COMPUTE => { let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; - check_compute(context, &cmd, &compute)?; + check_compute(&mut context, &cmd, &compute)?; let slots = resolved.get(index).ok_or(EINVAL)?; commands.push( Command::Compute(KBox::new( diff --git a/drivers/gpu/drm/asahi/g15/encode.rs b/drivers/gpu/drm/asahi/g15/encode.rs index e46ecd05bdf78b..64aebf32548914 100644 --- a/drivers/gpu/drm/asahi/g15/encode.rs +++ b/drivers/gpu/drm/asahi/g15/encode.rs @@ -64,10 +64,9 @@ const fn align(n: u64, a: u64) -> u64 { } /// Firmware stamp value of ordinal `n` on the queue with this seed. -/// Stamps step by 0x100 through 0xffffff values and never read 0. +/// Stamps step by 0x100, including zero when the counter wraps. pub(crate) const fn stamp_seed(seed: u32, n: u64) -> u32 { - let base = (seed / 0x100) as u64; - ((((base - 1 + n % 0xffffff) % 0xffffff) + 1) * 0x100) as u32 + seed.wrapping_add((n as u32).wrapping_mul(0x100)) } /// Compute stamp of ordinal `n`. diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs index 33974222afe9b0..80e4a31a78706d 100644 --- a/drivers/gpu/drm/asahi/g15/engine.rs +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -155,10 +155,10 @@ pub(crate) struct Flight { pub(crate) fs_start: u32, /// Channel producers (TA, FS, compute) after the prefix's records. pub(crate) rec: [u32; 3], - /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none): + /// Compute member j's FW stamp target, and the FS target it waits on directly: /// its predecessors in the prefix (for the compute-hang check). pub(crate) cstamp: [u32; sizes::COMPUTE_SLOTS], - pub(crate) cwait: [u32; sizes::COMPUTE_SLOTS], + pub(crate) cwait: [Option; sizes::COMPUTE_SLOTS], /// Compute slots, pass slots and clustering buffers it holds. pub(crate) claims: Claims, /// The prefix rebuilds the render allocators (priming); nothing is published behind it. @@ -189,7 +189,7 @@ impl Flight { fs_start: 0, rec: [0; 3], cstamp: [0; sizes::COMPUTE_SLOTS], - cwait: [0; sizes::COMPUTE_SLOTS], + cwait: [None; sizes::COMPUTE_SLOTS], claims: Claims::new(), rebuild: false, doomed: false, @@ -385,9 +385,11 @@ pub(crate) struct Engine { pub(crate) rebuild_live: bool, /// A ready prefix waits only for a record's consumption: repoll sooner. pub(crate) chan_wait: bool, + /// A ready job waits for an ASID held by a completing failed job. + pub(crate) slot_wait: bool, /// Progress tracking of the oldest prefix. pub(crate) progress_flight: u64, - pub(crate) progress: [u32; 3], + pub(crate) progress: [Option; 3], pub(crate) stats: Stats, /// Phase timers (debug_flags bit 49). pub(crate) prof: Prof, @@ -424,8 +426,9 @@ impl Engine { render_registered: false, rebuild_live: false, chan_wait: false, + slot_wait: false, progress_flight: 0, - progress: [0; 3], + progress: [None; 3], stats: Stats::default(), prof: Prof::default(), debugfs: None, @@ -546,6 +549,7 @@ impl Firmware { let mut result = Ok(()); self.engine.chan_wait = false; + self.engine.slot_wait = false; // Nothing is published while a hang request is armed. while self.recovery.failed().is_none() && !self.engine.frozen @@ -592,7 +596,7 @@ impl Firmware { dev_err!( self.graph.dev.as_ref(), "G15: publication of ctx {} failed after a queue WRITE ({:?})\n", - job.vm.slot, + job.slot(), e ); complete(done, &job, Err(EIO)); @@ -675,6 +679,15 @@ impl Firmware { /// yet (no free slot, or a channel record still unconsumed); EIO once a queue WRITE /// advanced; any other error means nothing was published. fn publish_prefix(&mut self, job: &Arc, done: &mut Completions) -> Result { + if let Err(error) = job.bind() { + if error == ENOSPC { + // A failed job can keep its binding until scheduler references drop after + // this pass. Leave healthy work pending and retry without the device lock. + self.engine.slot_wait = true; + return Err(EAGAIN); + } + return Err(error); + } let mut f = Flight::new(job.clone()); let result = self.publish_prefix_into(job, &mut f); match result { @@ -682,7 +695,11 @@ impl Firmware { // Never published: its storage is free again, and horizon jobs taken into it // fail with it. self.slots.release(&mut f.claims); + if job.progress.next() == job.progress.sent() { + job.release_binding(); + } for j2 in f.group.iter() { + j2.release_binding(); complete(done, j2, Err(e)); } } @@ -899,11 +916,22 @@ impl Firmware { hz_stop = Some(1); break; } - for k in 0..n2 { - match command(&j2, k).ok_or(EINVAL)? { - Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?, - Command::Render(_) => return Err(EINVAL), + j2.bind()?; + let encoded = (|| { + for k in 0..n2 { + match command(&j2, k).ok_or(EINVAL)? { + Command::Compute(c) => { + self.prefix_compute(&j2, c, &mut pf, f, false)? + } + Command::Render(_) => return Err(EINVAL), + } } + Ok(()) + })(); + if let Err(error) = encoded { + // Not in the horizon yet and nothing of it reached the firmware. + j2.release_binding(); + return Err(error); } computes += n2; j2.progress.set_sent(n2); @@ -1057,7 +1085,7 @@ impl Firmware { Queue, self.graph.dev, "G15: prefix ctx={} computes {} (n={} head {}) renders {} (draw {} ta/fs head {}/{}) behind {}: {:?}\n", - job.vm.slot, + job.slot(), computes, pf.n, pf.c_head, @@ -1095,7 +1123,7 @@ impl Firmware { ) -> Result { let n = pf.n + 1; let t = encode::stamp(n); - let ctx = job.vm.slot; + let ctx = job.slot(); let ts = &cmd.timestamps; let mut lap = Lap::start(); // Nothing published yet on failure. @@ -1191,11 +1219,7 @@ impl Firmware { } let j = pf.computes as usize; f.cstamp[j] = t; - f.cwait[j] = if pf.prev == Prev::Render { - pf.last_fs - } else { - 0 - }; + f.cwait[j] = (pf.prev == Prev::Render).then_some(pf.last_fs); g.ring.w64(8 * pf.c_head as usize, s.work.va)?; pf.c_head = (pf.c_head + 1) % sizes::QUEUE_CAP; lap.lap(&mut self.engine.prof.write_ns); @@ -1240,7 +1264,7 @@ impl Firmware { let k = f.claims.pass[pf.renders as usize] as usize; let n = pf.draw + 1; let usc = job.queue.usc_base; - let ctx = job.vm.slot; + let ctx = job.slot(); let tta = encode::stamp_seed(ids::TA_SEED, n); let tfs = encode::stamp_seed(ids::FS_SEED, n); // Early tiling: a later render member after a render of the same prefix whose render diff --git a/drivers/gpu/drm/asahi/g15/gate.rs b/drivers/gpu/drm/asahi/g15/gate.rs index a6fb20bf156b5c..590f5e6f0a5c11 100644 --- a/drivers/gpu/drm/asahi/g15/gate.rs +++ b/drivers/gpu/drm/asahi/g15/gate.rs @@ -71,7 +71,7 @@ struct Window { impl Window { /// The `count` words at `va`, reading whole spans of up to 256 bytes (not past the page /// unless the record crosses it). Unreadable memory: EFAULT. - fn words(&mut self, vm: &mmu::Vm, va: u64, count: usize) -> Result<[u32; 10]> { + fn words(&mut self, vm: &mut mmu::VmReadGuard, va: u64, count: usize) -> Result<[u32; 10]> { let n = count * 4; let cached = self.len != 0 && va >= self.va @@ -108,7 +108,7 @@ struct Checked { /// One page check of `[start, end)` under the walk's page budget: ENOSPC once the budget is /// spent. -fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result { +fn check(vm: &mut mmu::VmReadGuard, start: u64, end: u64, count: &mut u32) -> Result { if *count >= CDM_MAX_PAGE_CHECKS { return Err(ENOSPC); } @@ -118,7 +118,7 @@ fn check(vm: &mmu::Vm, start: u64, end: u64, count: &mut u32) -> Result { /// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable. /// The last proven program page is not checked again. -fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Result { +fn program(vm: &mut mmu::VmReadGuard, usc: u64, address: u64, checked: &mut Checked) -> Result { if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 { return Err(EINVAL); } @@ -134,7 +134,7 @@ fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Checked) -> Resul /// An indirect launch's 12 argument bytes: readable. The last proven argument page is not /// checked again; arguments that cross a page are checked each time. -fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result { +fn arguments(vm: &mut mmu::VmReadGuard, address: u64, checked: &mut Checked) -> Result { let end = address.checked_add(12).ok_or(EINVAL)?; let page = address & !(mmu::UAT_PGMSK as u64); let within = (end - 1) & !(mmu::UAT_PGMSK as u64) == page; @@ -149,7 +149,7 @@ fn arguments(vm: &mmu::Vm, address: u64, checked: &mut Checked) -> Result { } /// VDM admission: a normal root stream opens with exactly the G15 barrier. -pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { +pub(crate) fn vdm(vm: &mut mmu::VmReadGuard, va: u64) -> Result { if !params::stream_gate() { return Ok(()); } @@ -165,7 +165,7 @@ pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { /// CDM admission: walks the stream from its base through no-return links until TERMINATE, /// admitting only the known G15S records: direct launch (40), indirect launch (36/24), barriers /// (4), link (8). -pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { +pub(crate) fn cdm(vm: &mut mmu::VmReadGuard, usc: u64, base: u64) -> Result { if !params::stream_gate() { return Ok(()); } diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c index b087bb9ca954ee..b14b30aff3500c 100644 --- a/drivers/gpu/drm/asahi/g15/glue.c +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -35,6 +35,7 @@ void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size); bool asahi_g15_dma_coherent(struct device *dev); int asahi_g15_pmp_prepare(struct device *dev); int asahi_g15_pmp_get(struct device *dev); +int asahi_g15_pmp_ready(void); void asahi_g15_pmp_put(struct device *dev); bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); @@ -150,6 +151,11 @@ int asahi_g15_pmp_get(struct device *dev) } +int asahi_g15_pmp_ready(void) +{ + return apple_pmp_t6030_gpu_ready(); +} + /* * Give back the vote of asahi_g15_pmp_get() after a start that failed before * the firmware was touched. The full M3 stack's J514S bridge has no release: diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs index fd8771d9d3e65a..f231e573c40176 100644 --- a/drivers/gpu/drm/asahi/g15/hw.rs +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -514,8 +514,6 @@ pub(crate) mod sizes { pub(crate) const AHEAD_RECORDS: u32 = 64; /// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit). pub(crate) const MAX_QUEUES: u32 = 58; - /// TTBAT user slots: user VMs hold one each for their lifetime (ASIDs 1..63). - pub(crate) const MAX_VMS: u32 = 63; } /// Firmware channels, doorbells, seeds and identifiers. diff --git a/drivers/gpu/drm/asahi/g15/job.rs b/drivers/gpu/drm/asahi/g15/job.rs index a360b58987c815..ecbe1c8c991aa1 100644 --- a/drivers/gpu/drm/asahi/g15/job.rs +++ b/drivers/gpu/drm/asahi/g15/job.rs @@ -21,30 +21,36 @@ //! The engine owns an `Arc` from [`super::Firmware::enqueue`] on. It completes the job //! exactly once with [`Job::complete`] (later calls are ignored), through [`Completions`] //! collected under the device lock and [`complete_all`] after it is dropped. A job whose storage -//! the firmware may still reference stays in `Engine::retained` (failed jobs until reboot): its -//! [`mmu::VmJobGuard`] keeps the VM's mappings and its VM state keeps the VM's TTBAT slot until +//! the firmware may still reference after a fail-stop stays in `Engine::retained` until reboot: its +//! [`mmu::VmJobGuard`] keeps the VM's mappings and its binding keeps the VM's TTBAT slot until //! the last `Arc` drops. -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use kernel::{ bindings, c_str, device, dma_fence::{Fence, RawDmaFence}, drm::sched, + new_mutex, prelude::*, - sync::Arc, + sync::{Arc, Mutex}, }; use super::{command::Command, fence::JobFence, hw, queue::QueueState, vm::VmState, Shared}; use crate::mmu; /// A job as the engine sees it. +#[pin_data] pub(crate) struct Job { /// Submission ID (debugging). pub(crate) id: u64, /// The submitting queue (priority, service turn, horizon history). pub(crate) queue: Arc, - /// The job's VM (TTBAT slot, status). + /// The job's VM and status. pub(crate) vm: Arc, + /// Hardware context, assigned immediately before the engine publishes the job. + context: AtomicU32, + #[pin] + binding: Mutex>, /// Validated commands, in submission order (at most 64). pub(crate) commands: KVec, /// The job's hardware fence. @@ -73,11 +79,13 @@ impl Job { guard: mmu::VmJobGuard, ) -> Result> { let progress = super::engine::Progress::new(commands.len()); - Ok(Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { id, queue, vm, + context: AtomicU32::new(0), + binding <- new_mutex!(None, "g15::Job::binding"), commands, fence, progress, @@ -85,9 +93,29 @@ impl Job { _guard: guard, accepted: AtomicBool::new(false), started: AtomicBool::new(false), - }, + }), GFP_KERNEL, - )?) + ) + } + + pub(crate) fn slot(&self) -> u32 { + self.context.load(Ordering::Relaxed) + } + + /// Called under the device lock immediately before publication. + pub(crate) fn bind(&self) -> Result { + let mut slot = self.binding.lock(); + if slot.is_none() { + let binding = self.vm.try_bind()?; + self.context.store(binding.slot(), Ordering::Relaxed); + *slot = Some(binding); + } + Ok(()) + } + + /// All commands retired successfully, or nothing of the job was published. + pub(crate) fn release_binding(&self) { + drop(self.binding.lock().take()); } /// Completes the job: latches the VM status on failure, then signals the fence (after every @@ -120,6 +148,9 @@ impl Job { self.vm.status.record(error); } } + if result.is_ok() || !engine { + self.release_binding(); + } self.fence.signal(result); } diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index af77a54f9c189a..ea459e4e4ea5e0 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -375,7 +375,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. + /// Live VMs and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -539,7 +539,21 @@ impl Firmware { vms, }, ); - if let Err(e) = firmware.init_graphs() { + let result = firmware.init_graphs().and_then(|_| { + if cfg.platform != hw::Platform::AdtWithPmp { + return Ok(()); + } + // A report before firmware startup is ACKed without enabling bandwidth scaling. + // Reassert it after setup, including the backend's final device control. + // SAFETY: This session retains its acknowledged T6030 AGX power vote. + let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; + if ret == 0 { + Ok(()) + } else { + Err(Error::from_errno(ret)) + } + }); + if let Err(e) = result { let Firmware { session, init, diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs index 751bccd07b119e..647c503ba55037 100644 --- a/drivers/gpu/drm/asahi/g15/poll.rs +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -117,7 +117,7 @@ impl Shared { if let Err(error) = firmware.publish(&mut done) { self.fail_stop(error); } - if !firmware.engine.flights.is_empty() { + if !firmware.engine.flights.is_empty() || firmware.engine.slot_wait { self.rearm(firmware.engine.chan_wait); } } @@ -189,10 +189,7 @@ impl Shared { } } -/// Stamp cell `obs` reached its own target, bounded by the newest published target `tail`. -pub(crate) fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { - obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) -} +pub(crate) use super::recovery::logic::stamp_reached; /// Cursor `c` of a ring of `m` entries passed end `h` of entries from `b`, within tail `t`. pub(crate) fn ring_passed(c: u32, b: u32, h: u32, t: u32, m: u32) -> bool { @@ -218,7 +215,7 @@ impl Firmware { // Idle bookkeeping of the PB: when the idle began, and the reclaim once it is due. self.note_idle(); self.reclaim_check(); - if self.recovery_idle() { + if self.recovery_idle() && self.engine.pending.is_empty() { return Ok(false); } } @@ -243,7 +240,10 @@ impl Firmware { // at the start of a pass that finds the flights empty. self.note_idle(); published?; - Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen) + Ok(!self.engine.flights.is_empty() + || self.engine.slot_wait + || self.recovery.needs_repoll() + || self.engine.frozen) } /// Drains the rings and retires finished flights, with up to four trailing 5 us polls @@ -462,6 +462,10 @@ impl Firmware { p.count.store(p.sent(), Ordering::Relaxed); } } + if result.is_ok() && p.next() == p.sent() { + // All published prefixes retired; another VM may run before this job resumes. + job.release_binding(); + } if p.next() < p.sent() || p.sent() < p.count() { return; // more of it published or still to publish } @@ -479,6 +483,7 @@ impl Firmware { match result { Err(error) => complete(done, j2, Err(error)), Ok(()) => { + j2.release_binding(); s.jobs += 1; s.job_cmds += p2.count() as u64; s.job_max = s.job_max.max(p2.count() as u64); @@ -498,23 +503,25 @@ impl Firmware { return; }; let (g, r, e) = (&self.graph.compute, &self.graph.render, &self.engine); - let (mut c, mut ta, mut fs) = (0, 0, 0); + let pending = super::recovery::logic::pending_stamp; + let (mut c, mut ta, mut fs) = (None, None, None); if f.computes != 0 { - c = g.fw_stamp.r32(0).unwrap_or(0); - if stamp_reached(c, f.target, e.target) { - c = 0; // its computes are done: later work does not count - } + c = g + .fw_stamp + .r32(0) + .ok() + .and_then(|v| pending(v, f.target, e.target)); } if f.renders != 0 { - ta = r.fw_stamp[0].r32(0).unwrap_or(0); - fs = r.fw_stamp[1].r32(0).unwrap_or(0); - if stamp_reached(ta, f.ta_target, e.ta_target) { - ta = 0; - } - if stamp_reached(fs, f.fs_target, e.fs_target) { - fs = 0; - } + ta = r.fw_stamp[0] + .r32(0) + .ok() + .and_then(|v| pending(v, f.ta_target, e.ta_target)); + fs = r.fw_stamp[1] + .r32(0) + .ok() + .and_then(|v| pending(v, f.fs_target, e.fs_target)); } let moved = e.progress_flight == f.published_ns && [c, ta, fs] != e.progress; let published = f.published_ns; @@ -607,7 +614,7 @@ impl Firmware { self.graph.dev.as_ref(), "G15: {}: ctx={} prefixes={} n={} target={:#x} user={:#x} fw={:#x} done={} read={} write={} head={} ch={}/{} events={}/{} errors={} busy={} fault={:#x} {}ms\n", what, - f.map_or(0, |f| f.job.vm.slot), + f.map_or(0, |f| f.job.slot()), e.flights.len(), e.ordinal, e.target, diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs index 3364b136b9287f..8a3989f723abb7 100644 --- a/drivers/gpu/drm/asahi/g15/pools.rs +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -624,8 +624,8 @@ impl Firmware { if others != 0 { dev_err!( self.dev(), - "PB backing: ctx {} maps {} chunks with {} flights of another VM\n", - vm.slot, + "PB backing: VM {} maps {} chunks with {} flights of another VM\n", + vm.id, self.pools.chunks.len(), others ); @@ -913,7 +913,7 @@ impl Firmware { .flights .iter() .enumerate() - .any(|(i, f)| self.receipts(i).renders != 0 && f.job.vm.slot == ctx); + .any(|(i, f)| self.receipts(i).renders != 0 && f.job.slot() == ctx); // At the maximum, after the refusals, the request is left unanswered: the poll pass // returns, and the host deadline and its recovery path apply to a stalled TA. if self.pools.ledger.exhausted() { diff --git a/drivers/gpu/drm/asahi/g15/recovery.rs b/drivers/gpu/drm/asahi/g15/recovery.rs index 0827385e92c5fa..fc9bba6cbdce3b 100644 --- a/drivers/gpu/drm/asahi/g15/recovery.rs +++ b/drivers/gpu/drm/asahi/g15/recovery.rs @@ -224,7 +224,7 @@ impl Firmware { for flight in self.engine.flights.iter_mut().skip(1) { flight.doomed = true; } - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); let kind = if offset == logic::FRAGMENT_KILL { "fragment" } else { @@ -281,7 +281,7 @@ impl Firmware { if decision.compute_hang { self.report("compute hang"); } - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); if decision.class0_hang { dev_warn!( self.dev(), @@ -323,7 +323,7 @@ impl Firmware { /// anything is published; if it cannot be, the GPU stops. Anything else fail-stops. fn fail_or_recover(&mut self, recover: bool, fault: bool, done: &mut Completions) -> Result { let flight = !self.engine.flights.is_empty(); - let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.slot()); self.report(if flight { "prefix FAILED" } else { diff --git a/drivers/gpu/drm/asahi/g15/recovery/logic.rs b/drivers/gpu/drm/asahi/g15/recovery/logic.rs index 6d36d1d6550d5a..767fb7c9529372 100644 --- a/drivers/gpu/drm/asahi/g15/recovery/logic.rs +++ b/drivers/gpu/drm/asahi/g15/recovery/logic.rs @@ -37,9 +37,18 @@ pub(crate) const DRAIN_POLLS: u32 = 1000; pub(crate) const FURTHER_HALTS: u32 = 4; /// Stamp cell `obs` reached the own target `own`, bounded by the newest published target -/// `tail`. Stamps are seeded so that a written stamp is never 0 and has a zero low byte. +/// `tail`. Written stamps have a zero low byte, including at counter wrap. pub(crate) const fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { - obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) + obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) +} + +/// The position of unfinished work; zero is a valid position after stamp wrap. +pub(crate) const fn pending_stamp(obs: u32, own: u32, tail: u32) -> Option { + if stamp_reached(obs, own, tail) { + None + } else { + Some(obs) + } } /// Cursor `c` of a ring of `m` entries passed the end `h` of entries starting at `b`, within @@ -73,9 +82,9 @@ pub(crate) struct Receipts { pub(crate) fs_start: u32, /// Host channel producers (TA, FS, compute) after the prefix's records. pub(crate) rec: [u32; 3], - /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none). + /// Compute member j's FW stamp target, and the FS target it waits on directly. pub(crate) cstamp: [u32; 16], - pub(crate) cwait: [u32; 16], + pub(crate) cwait: [Option; 16], } /// The newest published receipts. @@ -136,8 +145,10 @@ pub(crate) fn compute_hang(f: &Receipts, t: &Tail, o: &Observed) -> bool { if j == computes { return false; } - if f.cwait[j] != 0 && !stamp_reached(o.fs_stamp, f.cwait[j], t.fs_target) { - return false; // still waiting on a render of the prefix + if let Some(target) = f.cwait[j] { + if !stamp_reached(o.fs_stamp, target, t.fs_target) { + return false; // still waiting on a render of the prefix + } } if o.consumer[COMPUTE] != f.rec[COMPUTE] { return false; // its record not consumed yet @@ -332,10 +343,20 @@ mod tests { assert!(!stamp_reached(stamp(4), stamp(1), stamp(3))); // Across the u32 wrap. let own = 0xffffff00u32; + assert!(stamp_reached(0, own, 0x00000200)); + assert!(stamp_reached(0, 0, 0)); + assert!(!stamp_reached(0xffffff00, 0, 0x100)); assert!(stamp_reached(0x00000100, own, 0x00000200)); assert!(!stamp_reached(0xfffffe00, own, 0x00000200)); } + #[test] + fn progress_at_zero_wrap() { + assert_eq!(pending_stamp(0, 0x100, 0x200), Some(0)); + assert_eq!(pending_stamp(0, 0, 0x100), None); + assert_eq!(pending_stamp(0xffffff00, 0, 0x100), Some(0xffffff00)); + } + #[test] fn rings() { // Entries 10..14 of a 0x500 ring, tail at 20. @@ -401,13 +422,19 @@ mod tests { assert!(!compute_hang(&f, &t, &o)); o.compute_read = 15; // The running compute waits on an FS of the prefix that has not completed. - f.cwait[1] = 0x3d000300; + f.cwait[1] = Some(0x3d000300); o.fs_stamp = 0x3d000200; let mut t2 = t; t2.fs_target = 0x3d000400; assert!(!compute_hang(&f, &t2, &o)); o.fs_stamp = 0x3d000300; assert!(compute_hang(&f, &t2, &o)); + f.cwait[1] = Some(0); + t2.fs_target = 0x100; + o.fs_stamp = 0xffffff00; + assert!(!compute_hang(&f, &t2, &o)); + o.fs_stamp = 0; + assert!(compute_hang(&f, &t2, &o)); // No compute in the prefix. f.computes = 0; assert!(!compute_hang(&f, &t, &o)); diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 7ab3a1919eaecc..f457b924cbb943 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,8 +25,10 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` -//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) +//! and return them after successful retirement. Failed published jobs retain their slots until +//! their last reference drops (until reboot for jobs retained after a fail-stop). +//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs index e2b9d884bc25b8..24a912d5968bb6 100644 --- a/drivers/gpu/drm/asahi/g15/vm.rs +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -13,16 +13,12 @@ //! whole PB range, whose backing chunks are mapped later, before the VM's prefixes are //! published (`Firmware::map_pb`) and, for the VM of the flights, when a chunk is backed //! ([`VmState::map_heap`], [`VmState::map_chunk`]); -//! - bind it to an exclusive TTBAT slot ([`mmu::Uat::bind`]) for as long as its [`VmState`] -//! lives; at most [`super::hw::sizes::MAX_VMS`] such VMs (ENOSPC beyond, checked before -//! binding so the slot allocator never blocks). +//! - leave it unbound until a runnable job needs a TTBAT slot. Each published job holds +//! its binding through completion; failed published jobs retain theirs until their last +//! reference drops (until reboot for jobs retained after a fail-stop). //! -//! The [`VmState`] of a VM is shared by the [`Registry`] (while the client's VM handle exists), -//! the VM's queues and its jobs. It cannot be owned by the [`mmu::Vm`] itself: the slot binding -//! holds a `Vm` reference. The registry drops its reference once the shared lifetime reports the -//! client's handle closed ([`mmu::Vm::is_closed`]), at the next VM creation; the slot returns when -//! the last queue or job of the VM is gone (failed jobs are retained until reboot, and their -//! VM's slot with them). +//! The registry keeps a VM's state while its client handle is open. Queues and jobs retain +//! closed VMs until their last reference drops. //! //! User page tables are cleaned to the point of coherency by the shared `pgtable.rs` (enabled by //! the adopted UAT); user objects are prepared at creation (`gpu::Gpu::prepare_new_objects`). @@ -30,7 +26,7 @@ //! alike. use core::ops::Range; -use core::sync::atomic::{AtomicU32, Ordering}; +use core::sync::atomic::Ordering; use kernel::{ drm::gem::BaseObject, new_mutex, @@ -39,7 +35,7 @@ use kernel::{ }; use super::{hw, object, Gpu}; -use crate::{debug::*, g17::status::VmStatus, gem, mmu, module_parameters}; +use crate::{debug::*, g17::status::VmStatus, gem, mmu}; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; @@ -123,20 +119,22 @@ struct HeapMappings { pub(crate) struct VmState { /// The `mmu::Vm` identifier (`Gpu::ids.vm`). pub(crate) id: u64, - /// The VM's TTBAT slot (ASID), for the work images' context. - pub(crate) slot: u32, /// The sticky status (shared with `file.rs`). pub(crate) status: Arc, /// The VM: span checks, stream reads and job guards. pub(crate) vm: mmu::Vm, - /// The exclusive slot binding, released before the live count drops. - bind: Option, + /// The adopted UAT, used to bind published jobs without waiting for a slot. + uat: Arc, registry: Arc, /// The PB heap chunks mapped in this VM. heap: Arc, } impl VmState { + pub(crate) fn try_bind(&self) -> Result { + self.uat.try_bind(&self.vm) + } + /// Drops the registry's references to closed VMs (the queues call this when destroyed). pub(crate) fn prune_registry(&self) { self.registry.prune(); @@ -199,39 +197,11 @@ impl VmState { } } -impl Drop for VmState { - fn drop(&mut self) { - mod_pr_debug!("G15: VM {} releases context {}\n", self.id, self.slot); - drop(self.bind.take()); - self.registry.live.fetch_sub(1, Ordering::AcqRel); - } -} - -/// Live-VM accounting and the VMs whose client handle is open. +/// The VMs whose client handle is open. #[pin_data] pub(crate) struct Registry { - /// VMs whose client handle may still be open, oldest first. #[pin] vms: Mutex>>, - /// Live [`VmState`]s (each holds a TTBAT slot) plus reservations being created. - live: AtomicU32, -} - -/// A reserved place among the live VMs, released unless a [`VmState`] takes it over. -struct Reservation<'a>(Option<&'a Registry>); - -impl Reservation<'_> { - fn into_state(mut self) { - self.0 = None; - } -} - -impl Drop for Reservation<'_> { - fn drop(&mut self) { - if let Some(registry) = self.0 { - registry.live.fetch_sub(1, Ordering::AcqRel); - } - } } impl Registry { @@ -239,32 +209,14 @@ impl Registry { Arc::pin_init( pin_init!(Self { vms <- new_mutex!(KVec::new(), "g15::Registry::vms"), - live: AtomicU32::new(0), }), GFP_KERNEL, ) } - /// Reserves a place for a new VM after dropping the registry's references to closed VMs. - fn reserve(&self) -> Result> { - self.prune(); - // The shared slot allocator hands out a single slot with robust isolation. - let limit = if *module_parameters::robust_isolation.value() != 0 { - 1 - } else { - hw::sizes::MAX_VMS - }; - self.live - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |live| { - (live < limit).then_some(live + 1) - }) - .map_err(|_| ENOSPC)?; - Ok(Reservation(Some(self))) - } - /// Drops the registry's references to VMs whose client handle is closed. Called on VM close /// ([`closed`]), on queue destruction and before a new VM is created: a closed VM's page - /// tables, bound objects and TTBAT slot go as soon as its last queue or job is gone. + /// tables and bound objects go as soon as its last queue or job is gone. fn prune(&self) { let mut closed = KVec::new(); { @@ -367,7 +319,7 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { ClientAlias::collect(&firmware.objects)?, ) }; - let reservation = registry.reserve()?; + registry.prune(); let id = gpu.ids.vm.next(); let vm = gpu.uat.new_vm(id, kernel_range)?.with_tracked_status()?; let status = vm.status().ok_or(EINVAL)?.clone(); @@ -380,25 +332,19 @@ pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { }), GFP_KERNEL, )?; - // The context names the root only now: everything above is already visible to the walker. - let bind = gpu.uat.bind(&vm)?; - let slot = bind.slot(); - // From here on the state (even one that fails to allocate) gives the place back. - reservation.into_state(); let state = Arc::new( VmState { id, - slot, status, vm: vm.clone(), - bind: Some(bind), + uat: gpu.uat.clone(), registry: registry.clone(), heap, }, GFP_KERNEL, )?; registry.vms.lock().push(state, GFP_KERNEL)?; - mod_pr_debug!("G15: VM {} context {}\n", id, slot); + mod_pr_debug!("G15: VM {} created\n", id); Ok(vm) } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 2bf4a6823024de..3a7941b4a36534 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -358,6 +358,20 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } +#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] +pub(crate) struct VmReadGuard(VmExecGuard); + +#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] +impl VmReadGuard { + pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { + Vm::covers_range_locked(&mut self.0, address, size, read, write) + } + + pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { + Vm::read_user_locked(&mut self.0, va, buf) + } +} + /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, @@ -1804,7 +1818,16 @@ impl Vm { /// mappings cannot change during the copy (GPUVM exec lock). #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - let mut inner = lock_vm(&self.gpuvm, None, false)?; + self.read_lock()?.read_user(va, buf) + } + + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + pub(crate) fn read_lock(&self) -> Result { + Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) + } + + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -2754,6 +2777,15 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, true) + } + + /// Binds a VM without waiting for another slot owner to retire. + pub(crate) fn try_bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, false) + } + + fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { let mut binding = vm.binding.lock(); if binding.binding.is_none() { @@ -2767,7 +2799,11 @@ impl Uat { None }); - let slot = self.slots.get(binding.bind_token)?; + let slot = if wait { + self.slots.get(binding.bind_token)? + } else { + self.slots.try_get(binding.bind_token)? + }; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs index fde7470fe57791..a504e7491bfda6 100644 --- a/drivers/gpu/drm/asahi/slotalloc.rs +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -216,6 +216,20 @@ impl SlotAllocator { &self, token: Option, cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + ) -> Result> { + self.get_inner_impl(token, cb, true) + } + + /// Gets a slot without waiting; returns ENOSPC when every slot is held. + pub(crate) fn try_get(&self, token: Option) -> Result> { + self.get_inner_impl(token, |_a, _b| Ok(()), false) + } + + fn get_inner_impl( + &self, + token: Option, + cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + wait: bool, ) -> Result> { let mut inner = self.0.inner.lock(); @@ -256,6 +270,9 @@ impl SlotAllocator { } if oldest_time == u64::MAX { + if !wait { + return Err(ENOSPC); + } if first && inner.slot_limit == usize::MAX { pr_warn!( "{}: out of slots, blocking\n", diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index d39f6759c5d56c..98d96d08f8a434 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -1292,6 +1292,62 @@ static int apple_pmp_report_set_state(struct apple_pmp_report *rep, u32 id, 50000); } +/* Find and check the report before taking the GPU vote; optionally update it. */ +static int apple_pmp_report_t6030_gfx(struct device *supplier, const bool *enable) +{ + struct apple_pmp_report *rep, *found = NULL; + struct device_node *pmp; + const u8 *table; + int len = 0, ret = -ENODEV; + + if (!IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) || + !of_machine_is_compatible("apple,t6030") || + !of_device_is_compatible(supplier->of_node, "apple,t6030-pmp-v2")) + return -ENODEV; + + /* T6030's ordered soc-device ID 5 is the acknowledged AGX request. */ + table = of_get_property(supplier->of_node, "apple,tunable-soc-device", &len); + if (!apple_pmp_devices_valid(table, len, BIT_ULL(4), BIT_ULL(4), true) || + memcmp(table + 4 * PMP_SOC_DEVICE_SIZE + 116, "AGX\0\0\0\0\0", 8)) + return -EINVAL; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->offsets != &apple_pmp_offsets_t6030) + continue; + pmp = of_parse_phandle(rep->dev->of_node, "apple,pmp", 0); + if (pmp == supplier->of_node) { + of_node_put(pmp); + if (found) + goto out; + found = rep; + } else { + of_node_put(pmp); + } + } + if (found) { + if (readq(found->base + found->offsets->status) != PMP_REPORT_READY) + ret = -EAGAIN; + else + ret = enable ? apple_pmp_report_set_state(found, 4, *enable, false) : 0; + } +out: + rcu_read_unlock(); + return ret; +} +int apple_pmp_report_check_t6030_gfx(struct device *supplier) +{ + return apple_pmp_report_t6030_gfx(supplier, NULL); +} +EXPORT_SYMBOL_GPL(apple_pmp_report_check_t6030_gfx); + +/* The RTKit AGX vote alone does not enable PMP's GPU bandwidth controllers. */ +int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) +{ + return apple_pmp_report_t6030_gfx(supplier, &enable); +} +EXPORT_SYMBOL_GPL(apple_pmp_report_set_t6030_gfx); + static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) { struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); diff --git a/drivers/soc/apple/pmp-export.c b/drivers/soc/apple/pmp-export.c index 4162d4615d1a63..09804b23b760dc 100644 --- a/drivers/soc/apple/pmp-export.c +++ b/drivers/soc/apple/pmp-export.c @@ -58,6 +58,7 @@ EXPORT_SYMBOL_GPL(apple_pmp_link_device); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) { + bool report_gfx; int ret; if ((command != 0x0e && command != 0x0f) || device_id > 15 || enabled > 1) @@ -69,6 +70,43 @@ int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled) ret = apple_pmp_report_wait_supplier_ready(pmp_device, 5000); if (ret) return ret; - return apple_pmp_send_power_command(pmp_data, command, device_id, enabled); + /* T6030 needs both the RTKit power vote and the PTD bandwidth report. */ + report_gfx = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && + command == 0x0f && device_id == 5 && + of_machine_is_compatible("apple,t6030") && + of_device_is_compatible(pmp_device->of_node, "apple,t6030-pmp-v2"); + if (report_gfx) { + ret = apple_pmp_report_check_t6030_gfx(pmp_device); + if (ret) + return ret; + } + if (report_gfx && !enabled) { + ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); + if (ret) { + /* Clearing was written before the ACK timed out. Restore it. */ + apple_pmp_report_set_t6030_gfx(pmp_device, true); + return ret; + } + } + ret = apple_pmp_send_power_command(pmp_data, command, device_id, enabled); + /* Power is unknown after a lost reply; assert the report conservatively. */ + if (report_gfx && !enabled && ret) + apple_pmp_report_set_t6030_gfx(pmp_device, true); + return ret; } EXPORT_SYMBOL_GPL(apple_pmp_set_device_power); + +/* Reassert after startup, including a report left set by an earlier failure. */ +int apple_pmp_t6030_gpu_ready(void) +{ + int ret; + + guard(mutex)(&pmp_lock); + if (!pmp_data) + return -ENODEV; + ret = apple_pmp_report_set_t6030_gfx(pmp_device, false); + if (ret) + return ret; + return apple_pmp_report_set_t6030_gfx(pmp_device, true); +} +EXPORT_SYMBOL_GPL(apple_pmp_t6030_gpu_ready); diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index 1bfb4fc69f1f7a..1d0689f31c2039 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -15,6 +15,9 @@ struct device; #if IS_ENABLED(CONFIG_APPLE_PMP_REPORT) int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms); +/* Caller holds the PMP owner's vote lock and keeps the supplier bound. */ +int apple_pmp_report_check_t6030_gfx(struct device *supplier); +int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable); int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort); @@ -25,6 +28,16 @@ static inline int apple_pmp_report_wait_supplier_ready(struct device *supplier, return -ENODEV; } +static inline int apple_pmp_report_check_t6030_gfx(struct device *supplier) +{ + return -ENODEV; +} + +static inline int apple_pmp_report_set_t6030_gfx(struct device *supplier, bool enable) +{ + return -ENODEV; +} + static inline int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout) { diff --git a/include/linux/soc/apple/pmp.h b/include/linux/soc/apple/pmp.h index 2317ed46f42de1..8b329bf5d4dd5e 100644 --- a/include/linux/soc/apple/pmp.h +++ b/include/linux/soc/apple/pmp.h @@ -8,6 +8,8 @@ struct device; #if IS_REACHABLE(CONFIG_APPLE_PMP) int apple_pmp_link_device(struct device *consumer); int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enabled); +/* After the acknowledged AGX vote and successful GPU firmware startup. */ +int apple_pmp_t6030_gpu_ready(void); #else static inline int apple_pmp_link_device(struct device *consumer) { @@ -17,6 +19,10 @@ static inline int apple_pmp_set_device_power(u8 command, u16 device_id, u32 enab { return -ENODEV; } +static inline int apple_pmp_t6030_gpu_ready(void) +{ + return -ENODEV; +} #endif /* Registration is private to the PMP Rust driver and its C bridge. */ int apple_pmp_register(struct device *dev, const void *data); diff --git a/tools/asahi/display26-tests.py b/tools/asahi/display26-tests.py index 8d2e497fdcf649..2366766042dc2f 100644 --- a/tools/asahi/display26-tests.py +++ b/tools/asahi/display26-tests.py @@ -96,6 +96,7 @@ def main(): #include #include #include +#include #include struct device_node { int id; }; struct device { struct device_node *of_node; bool bound; int refs; }; struct device_link { int unused; }; @@ -117,12 +118,17 @@ def main(): static struct device *get_device(struct device*d){d->refs++;return d;} static void put_device(struct device*d){d->refs--;} static struct device_link *device_link_add(struct device*c,struct device*s,unsigned flags){static struct device_link l;assert(flags==1);return link_fail?NULL:&l;} +/* This harness exercises the J613/25G83 bridge, not the T6030 GFX report. */ +static bool of_machine_is_compatible(const char *name){return !strcmp(name,"apple,t8132");} +static bool of_device_is_compatible(const struct device_node*n,const char*name){return false;} ''') write(tmp, 'bridge.c', '''#include #include "drivers/soc/apple/pmp-export.c" static int ready_result,send_result,calls; int apple_pmp_report_wait_supplier_ready(struct device*d,unsigned timeout){assert(d==pmp_device);assert(timeout==5000);return ready_result;} int apple_pmp_send_power_command(const void*data,u8 command,u16 id,u32 enabled){assert(data==pmp_data);calls++;return send_result;} +int apple_pmp_report_check_t6030_gfx(struct device*d){assert(!"unexpected T6030 report on J613");return -EIO;} +int apple_pmp_report_set_t6030_gfx(struct device*d,bool enabled){assert(!"unexpected T6030 report on J613");return -EIO;} int main(void){ struct device_node n={1},wrong={2}; struct device supplier={&n,true,0},consumer={NULL,false,0}; int data; assert(apple_pmp_set_device_power(0xf,8,1)==-ENODEV); diff --git a/tools/asahi/g15-input-error-tests.py b/tools/asahi/g15-input-error-tests.py index 47971fccfa7106..80103e1bcad0e6 100644 --- a/tools/asahi/g15-input-error-tests.py +++ b/tools/asahi/g15-input-error-tests.py @@ -2,8 +2,10 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT """Actual G15 job dependency completion and G17 status/policy bodies on host services. -DMA fence lookup, engine submission, queue-fence ordering and kernel mutexes are -models. No kernel scheduling, GPU access or firmware execution is exercised. +DMA fence lookup, engine submission, queue-fence ordering, VM slot allocation and +kernel mutexes are models. Production job binding/completion and prefix admission +bodies run against them. No kernel scheduling, GPU access or firmware execution +is exercised. """ import argparse import hashlib @@ -33,6 +35,7 @@ def extract(text, marker): type Result = std::result::Result; const EIO:Error=Error(-5);const ENODEV:Error=Error(-19);const ETIMEDOUT:Error=Error(-110); const ECANCELED:Error=Error(-125);const EINVAL:Error=Error(-22);const ENOMEM:Error=Error(-12); +const ENOSPC:Error=Error(-28);const EAGAIN:Error=Error(-11); macro_rules! pr_warn { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } struct Mutex(StdMutex);impl Mutex{fn lock(&self)->MutexGuard<'_,T>{self.0.lock().unwrap()}} struct Mirror{word:*mut i32} @@ -44,8 +47,17 @@ def extract(text, marker): mod g17 {use super::*;__POLICY__} mod bindings {pub unsafe fn dma_fence_get_status(p:*const i32)->i32{unsafe{*p}}} struct Fence(i32);impl Fence{fn raw(&self)->*const i32{&self.0}} -struct Vm{status:VmStatus} -struct Progress{signaled:AtomicBool} +struct Binding{slot:u32,drops:Arc} +impl Binding{fn slot(&self)->u32{self.slot}} +impl Drop for Binding{fn drop(&mut self){self.drops.fetch_add(1,Ordering::SeqCst);}} +struct Vm{status:VmStatus,refuse_binding:AtomicBool,next_slot:AtomicU32,allocations:AtomicU32,binding_drops:Arc} +impl Vm{fn try_bind(&self)->Result{ + if self.refuse_binding.load(Ordering::SeqCst){return Err(ENOSPC);} + self.allocations.fetch_add(1,Ordering::SeqCst); + Ok(Binding{slot:self.next_slot.load(Ordering::SeqCst),drops:self.binding_drops.clone()}) +}} +struct Progress{signaled:AtomicBool,next:AtomicU32,sent:AtomicU32} +impl Progress{fn next(&self)->u32{self.next.load(Ordering::SeqCst)}fn sent(&self)->u32{self.sent.load(Ordering::SeqCst)}} struct Guard{quarantines:AtomicU32}impl Guard{fn quarantine(&self){self.quarantines.fetch_add(1,Ordering::SeqCst);}} struct JobFence{vm:Arc,signals:AtomicU32,result:AtomicI32,saw_sticky:AtomicI32,saw_report:AtomicI32,saw_mirror:AtomicI32} impl JobFence{ @@ -60,12 +72,25 @@ def extract(text, marker): } struct Shared{crashed:AtomicBool,submits:AtomicU32,submit_result:Result} impl Shared{fn submit(&self,_:Arc)->Result{self.submits.fetch_add(1,Ordering::SeqCst);self.submit_result}} -struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence} +struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence,context:AtomicU32,binding:Mutex>} impl Job {__JOB__} +type Completions=Vec; +fn complete(done:&mut Completions,_job:&Arc,result:Result){done.push(result);} +struct Flight{claims:(),group:Vec>} +impl Flight{fn new(_job:Arc)->Self{Self{claims:(),group:vec![]}}} +struct Slots{releases:u32} +impl Slots{fn release(&mut self,_claims:&mut ()){self.releases+=1;}} +struct Engine{slot_wait:bool} +struct HostFirmware{engine:Engine,slots:Slots,publish_result:Result,publishes:u32} +impl HostFirmware{ + fn new(result:Result)->Self{Self{engine:Engine{slot_wait:false},slots:Slots{releases:0},publish_result:result,publishes:0}} + fn publish_prefix_into(&mut self,_job:&Arc,_flight:&mut Flight)->Result{self.publishes+=1;self.publish_result} + __PREFIX__ +} fn make_job(own:Option)->(Arc,Box){ - let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast())}); + let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast()),refuse_binding:AtomicBool::new(false),next_slot:AtomicU32::new(7),allocations:AtomicU32::new(0),binding_drops:Arc::new(AtomicU32::new(0))}); if let Some(error)=own{vm.status.record(error);} - let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false),next:AtomicU32::new(0),sent:AtomicU32::new(0)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)},context:AtomicU32::new(0),binding:Mutex(StdMutex::new(None))}); (job,mirror) } fn shared(crashed:bool,result:Result)->Arc{Arc::new(Shared{crashed:AtomicBool::new(crashed),submits:AtomicU32::new(0),submit_result:result})} @@ -89,7 +114,7 @@ def extract(text, marker): assert_eq!(job.fence.signals.load(Ordering::SeqCst),1,"dependency completion must signal once"); assert_eq!(job.vm.status.get(),0,"duplicate completion must not poison VM"); // A later job on the same VM is admitted despite the reported client error. - let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false),next:AtomicU32::new(0),sent:AtomicU32::new(0)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)},context:AtomicU32::new(0),binding:Mutex(StdMutex::new(None))}); later.start(&device,&[Fence(1)]);assert_eq!(device.submits.load(Ordering::SeqCst),1,"successful dependencies after report must submit"); } for own in [EIO,EINVAL]{for input in [ECANCELED,EIO,ENODEV,ETIMEDOUT,EINVAL]{ @@ -110,8 +135,70 @@ def extract(text, marker): } // An engine-owned fault still quarantines before completion. let(job,_mirror)=make_job(None);job.complete(Err(EIO));assert_eq!(job._guard.quarantines.load(Ordering::SeqCst),1);assert_eq!(job.vm.status.get(),EIO.to_errno()); + // Actual bind/release bodies: a job keeps its slot until explicitly released. + { + let(job,_mirror)=make_job(None); + job.vm.refuse_binding.store(true,Ordering::SeqCst); + assert_eq!(job.bind(),Err(ENOSPC));assert!(job.binding.lock().is_none()); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),0); + job.vm.refuse_binding.store(false,Ordering::SeqCst); + job.bind().unwrap();assert_eq!(job.slot(),7,"binding must publish its slot"); + job.vm.next_slot.store(9,Ordering::SeqCst); + job.bind().unwrap();assert_eq!(job.slot(),7,"repeated bind must keep existing binding"); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),1); + job.release_binding();job.release_binding(); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),1,"release_binding must drop exactly once"); + job.bind().unwrap();assert_eq!(job.slot(),9,"released job may acquire a different slot"); + assert_eq!(job.vm.allocations.load(Ordering::SeqCst),2); + } + // An engine failure must not recycle an ASID while retained job references live. + for result in [Ok(()),Err(EIO),Err(ETIMEDOUT),Err(ECANCELED)]{ + let(job,_mirror)=make_job(None);job.bind().unwrap();let drops=job.vm.binding_drops.clone(); + job.complete(result); + if result.is_err(){ + assert!(job.binding.lock().is_some(),"failed engine completion must retain binding"); + assert_eq!(drops.load(Ordering::SeqCst),0); + job.complete(Ok(()));assert!(job.binding.lock().is_some(),"duplicate completion must retain failed binding"); + let retained=job.clone();drop(job); + assert_eq!(drops.load(Ordering::SeqCst),0,"a retained reference must keep the binding"); + drop(retained);assert_eq!(drops.load(Ordering::SeqCst),1,"last reference releases failed binding"); + }else{ + assert!(job.binding.lock().is_none(),"successful engine completion releases binding"); + assert_eq!(drops.load(Ordering::SeqCst),1);drop(job);assert_eq!(drops.load(Ordering::SeqCst),1); + } + } + // Run the real prefix admission wrapper; only firmware publication is modeled. + { + let(job,_mirror)=make_job(None);let mut fw=HostFirmware::new(Ok(()));let mut done=vec![]; + job.vm.refuse_binding.store(true,Ordering::SeqCst); + assert_eq!(fw.publish_prefix(&job,&mut done),Err(EAGAIN)); + assert!(fw.engine.slot_wait,"slot exhaustion must request retry"); + assert_eq!(fw.publishes,0);assert!(done.is_empty());assert_eq!(job.vm.status.get(),0); + job.vm.refuse_binding.store(false,Ordering::SeqCst); + assert_eq!(fw.publish_prefix(&job,&mut done),Ok(()));assert_eq!(fw.publishes,1); + } + for published in [false,true]{ + let(job,_mirror)=make_job(None);job.bind().unwrap(); + job.progress.sent.store(u32::from(published),Ordering::SeqCst); + let mut fw=HostFirmware::new(Err(ENOMEM));let mut done=vec![]; + assert_eq!(fw.publish_prefix(&job,&mut done),Err(ENOMEM)); + assert_eq!(fw.slots.releases,1);assert!(done.is_empty()); + if published{ + assert!(job.binding.lock().is_some(),"published prefix must retain binding on append failure"); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),0); + }else{ + assert!(job.binding.lock().is_none(),"unpublished failure must release binding"); + assert_eq!(job.vm.binding_drops.load(Ordering::SeqCst),1); + } + } + { + let(job,_mirror)=make_job(None);let mut fw=HostFirmware::new(Err(EIO));let mut done=vec![]; + assert_eq!(fw.publish_prefix(&job,&mut done),Err(EIO)); + assert!(job.binding.lock().is_some(),"firmware-visible failure must retain binding"); + assert_eq!(fw.slots.releases,0); + } assert_eq!(unrelated.vm.status.get(),0,"unrelated VM untouched");assert_eq!(unrelated.vm.status.reported_error(),0);assert_eq!(other_mirror.load(Ordering::SeqCst),0);assert_eq!(unrelated.fence.signals.load(Ordering::SeqCst),0); - println!("PASS: actual Job::start/completion and VmStatus/policy: healthy/failed VMs, mirror ordering, one-shot completion, successful input, device faults and isolation"); + println!("PASS: production job/prefix/status bodies: input errors, binding reuse/release/retention, exhaustion retry, published-prefix lifetime and isolation"); } ''' @@ -129,16 +216,24 @@ def main(): current = (ROOT / job_path).read_text() status = (ROOT / "drivers/gpu/drm/asahi/g17/status.rs").read_text() policy = (ROOT / "drivers/gpu/drm/asahi/g17/dependency.rs").read_text() + engine = (ROOT / "drivers/gpu/drm/asahi/g15/engine.rs").read_text() methods = [extract(job, " pub(crate) fn complete("), extract(job, " fn finish("), extract(job, " fn start(self: &Arc")] # The old control needs the current helper for the duplicate-completion test; # it still runs its actual unmodified start, which reproduces the lost input error. methods.append(extract(job if "fn finish_dependency(" in job else current, " fn finish_dependency(")) + # Historical removed-fix controls predate publication bindings. Keep their + # actual start/completion bodies and use current binding helpers alongside. + binding_source = job if "fn release_binding(" in job else current + methods.extend(extract(binding_source, marker) for marker in [ + " pub(crate) fn slot(", " pub(crate) fn bind(", + " pub(crate) fn release_binding("]) bodies = "\n".join(methods) code = PRELUDE.replace("__JOB__", bodies).replace("__STATUS__", "\n".join( extract(status, marker) for marker in [" pub(crate) fn get(", " pub(crate) fn record(", " pub(crate) fn reported_error(", " pub(crate) fn report_failure("])).replace( - "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")) + "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")).replace( + "__PREFIX__", extract(engine, " fn publish_prefix(")) variants = {"control": code} expected = {} if args.mutations: @@ -152,6 +247,11 @@ def main(): "success-refused": ("if let Some(status) = failed {", "if let Some(status) = failed.or(Some(-125)) {", "successful dependencies after report must submit"), "dependency-completes-twice": (" fn finish_dependency(&self, error: Error) {\n if self.progress.signaled.swap(true, Ordering::AcqRel) {\n return;\n }", " fn finish_dependency(&self, error: Error) {", "dependency completion must signal once"), "mirror-not-published": ("unsafe { AtomicI32::from_ptr(mirror.word) }.store(error, Ordering::SeqCst);", "let _ = mirror;", "mirror must precede fence signal"), + "failed-engine-releases-binding": ("if result.is_ok() || !engine {", "if true {", "failed engine completion must retain binding"), + "binding-release-leaks": ("drop(self.binding.lock().take());", "let _ = self;", "release_binding must drop exactly once"), + "bound-job-rebinds": ("if slot.is_none() {", "if true {", "repeated bind must keep existing binding"), + "slot-exhaustion-not-retried": ("self.engine.slot_wait = true;", "self.engine.slot_wait = false;", "slot exhaustion must request retry"), + "published-prefix-releases-binding": ("if job.progress.next() == job.progress.sent() {", "if true {", "published prefix must retain binding on append failure"), } for name, (old, new, message) in changes.items(): if code.count(old) != 1: @@ -160,7 +260,8 @@ def main(): expected[name] = message variants["control-after"] = code receipt = {"scope": __doc__, "ref": args.ref, "sources": {name: hashlib.sha256(text.encode()).hexdigest() - for name, text in [(job_path, job), ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} + for name, text in [(job_path, job), ("g15/engine.rs", engine), + ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} for name, source in variants.items(): path = args.output / (name + ".rs"); path.write_text(source) binary = args.output / name diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py index e7f350d5c7f877..883a8d48a02d53 100644 --- a/tools/asahi/g15-pb-growth-tests.py +++ b/tools/asahi/g15-pb-growth-tests.py @@ -624,6 +624,7 @@ def consts(text, names): pub list: Mutex>, } pub struct VmState { + pub id: u64, pub slot: u32, pub vm: mmu::Vm, pub heap: Arc, @@ -631,6 +632,8 @@ def consts(text, names): impl VmState { pub fn new(slot: u32) -> Arc { Arc::new(VmState { + // Keep VM identity distinct from the modeled hardware slot. + id: 0x10000 + u64::from(slot), slot, vm: mmu::Vm::new(slot), heap: Arc::new(HeapMappings { list: Mutex::new(KVec(Vec::new())) }), @@ -640,6 +643,11 @@ def consts(text, names): pub struct Job { pub vm: Arc, } + impl Job { + pub fn slot(&self) -> u32 { + self.vm.slot + } + } pub struct Flight { pub job: Job, } diff --git a/tools/asahi/g15-retention-tests.py b/tools/asahi/g15-retention-tests.py index 4812a32b0b1e3e..5a4838ffc6be35 100644 --- a/tools/asahi/g15-retention-tests.py +++ b/tools/asahi/g15-retention-tests.py @@ -34,6 +34,8 @@ def item(text, marker): use std::{cell::{Cell, RefCell, RefMut}, mem::MaybeUninit, ops::{Deref, DerefMut}, sync::{Arc, atomic::{AtomicBool, Ordering}}}; type Error = i32; +trait FromErrno { fn from_errno(value: i32) -> Self; } +impl FromErrno for Error { fn from_errno(value: i32) -> Self { value } } type Result = std::result::Result; const GFP_KERNEL: u32 = 0; const EINVAL: Error = -22; @@ -43,6 +45,7 @@ def item(text, marker): static FAIL: Cell = const { Cell::new(0) }; static OWNER_DROPS: Cell = const { Cell::new(0) }; static BUFFER_DROPS: Cell = const { Cell::new(0) }; + static PMP_CALLS: Cell = const { Cell::new(0) }; } fn step(site: u32) -> Result { if FAIL.get() & (1 << site) != 0 { Err(-100 - site as i32) } else { Ok(()) } @@ -72,8 +75,13 @@ def item(text, marker): impl DerefMut for KBox { fn deref_mut(&mut self) -> &mut T { &mut self.0 } } struct AsahiDevice; impl AsahiDevice { fn as_ref(&self) -> &Self { self } } -struct Config; -static CONFIG: Config = Config; +mod hw { + #[derive(PartialEq)] + pub enum Platform { Adt, AdtWithPmp } +} +struct Config { platform: hw::Platform } +static CONFIG: Config = Config { platform: hw::Platform::Adt }; +static PMP_CONFIG: Config = Config { platform: hw::Platform::AdtWithPmp }; mod mmu { pub struct Uat; } mod regs { pub struct Identity; @@ -152,6 +160,12 @@ def item(text, marker): } mod boot { use super::*; + pub mod glue { + pub unsafe fn asahi_g15_pmp_ready() -> i32 { + super::PMP_CALLS.set(super::PMP_CALLS.get() + 1); + match super::step(13) { Ok(()) => 0, Err(e) => e } + } + } use super::{object::Objects, initdata::InitData, channel::Channels, event::Events}; pub struct Session { _owner: Owner, uat: Arc, regs: regs::Regs } impl Session { @@ -185,7 +199,9 @@ def item(text, marker): let boot::Booted { session, init, objects, channels, events, .. } = booted(); boot::Orphan { session, _objects: objects, channels: Some(channels), _init: Some(init), events } } -fn reset(fail: u32) { FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); } +fn reset(fail: u32) { + FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); PMP_CALLS.set(0); +} fn retained(label: &str) { assert_eq!(OWNER_DROPS.get(), 0, "{label}: live session owner dropped"); assert_eq!(BUFFER_DROPS.get(), 0, "{label}: firmware buffer dropped"); @@ -221,17 +237,32 @@ def item(text, marker): std::mem::forget(shared); } } - reset(0); + for orphan_oom in [false, true] { + reset((1 << 13) | if orphan_oom { 1 << 7 } else { 0 }); + let shared = Shared::new(); + let result = Firmware::start_backend(&AsahiDevice, &PMP_CONFIG, booted(), &shared); + assert!(matches!(result, Err(-113)), "PMP readiness error changed"); + assert_eq!(PMP_CALLS.get(), 1, "PMP readiness was not reported"); + retained("PMP readiness failure"); + assert_eq!(shared.orphan.lock().is_some(), !orphan_oom); + std::mem::forget(shared); + } + for cfg in [&CONFIG, &PMP_CONFIG] { + // A pending PMP error must be irrelevant on a platform without PMP. + reset(if cfg.platform == hw::Platform::Adt { 1 << 13 } else { 0 }); let shared = Shared::new(); let booted = boot::start(&shared).expect("normal boot"); assert_eq!(booted.max_frequency_khz, 1234); let uat: Arc = booted.session.uat(); - let firmware = Firmware::start_backend(&AsahiDevice, &CONFIG, booted, &shared).expect("normal backend"); + let firmware = Firmware::start_backend(&AsahiDevice, cfg, booted, &shared).expect("normal backend"); + assert_eq!(PMP_CALLS.get(), u32::from(cfg.platform == hw::Platform::AdtWithPmp), + "PMP readiness used on the wrong platform"); assert!(Arc::ptr_eq(&uat, &firmware.session.uat())); assert!(!shared.orphaned.load(Ordering::Acquire)); retained("normal success"); std::mem::forget((shared, firmware)); - println!("PASS 23 retention cases: orphan OOM, six backend failure sites, four boot-completion sites and normal success"); + } + println!("PASS 26 retention cases: orphan OOM, six backend failure sites, four boot-completion sites, PMP readiness failure and both platform success paths"); println!("normal success owner_drops={} buffer_drops={}", OWNER_DROPS.get(), BUFFER_DROPS.get()); } ''' @@ -269,11 +300,16 @@ def main(): cwd=ROOT, text=True) variants["consume-orphan-before-allocation"] = source.replace(orphan, item(old, " pub(crate) fn orphan(")) variants["unguarded-registry-allocation"] = source.replace(backend, item(old, " fn start_backend(")) + variants["ignore-pmp-readiness-error"] = source.replace( + "let ret = unsafe { boot::glue::asahi_g15_pmp_ready() };", + "let _ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; let ret = 0;") + variants["pmp-on-other-platform"] = source.replace( + "if cfg.platform != hw::Platform::AdtWithPmp {", "if false {") receipt = {"scope": __doc__, "sources": { name: hashlib.sha256(text.encode()).hexdigest() for name, text in [("mod.rs", mod), ("boot.rs", boot), ("kbox.rs", kbox)]}, "rustc": subprocess.check_output([os.environ.get("HOSTRUSTC", "rustc"), "--version"], text=True).strip(), - "cases": 23, "public_old_source_control": CONTROL_REF, "variants": {}} + "cases": 26, "public_old_source_control": CONTROL_REF, "variants": {}} for name, text in variants.items(): if name != "actual" and text == source: raise RuntimeError("NOT_APPLIED " + name) @@ -290,14 +326,20 @@ def main(): raise RuntimeError(result.stderr) else: expected = {"consume-orphan-before-allocation": "orphan allocation: live session owner dropped", - "unguarded-registry-allocation": "backend failure: live session owner dropped"}[name] - if result.returncode == 0 or expected not in result.stderr or "left: 5" not in result.stderr: + "unguarded-registry-allocation": "backend failure: live session owner dropped", + "ignore-pmp-readiness-error": "PMP readiness error changed", + "pmp-on-other-platform": "normal backend: -113"}[name] + if result.returncode == 0 or expected not in result.stderr: raise RuntimeError("INVALID mutant outcome " + name + "\n" + result.stderr) - receipt["variants"][name]["intended_owner_drops"] = 5 + if name in ("consume-orphan-before-allocation", "unguarded-registry-allocation"): + if "left: 5" not in result.stderr: + raise RuntimeError("INVALID owner-drop count " + name) + receipt["variants"][name]["intended_owner_drops"] = 5 + receipt["variants"][name]["intended_assertion"] = expected after = subprocess.run([str(out / "actual")], capture_output=True, text=True, check=True) receipt["actual_after_mutants"] = {"exit": after.returncode, "stdout": after.stdout, "stderr": after.stderr} receipt["fixed_invariants"] = {"live_session_owner_drops": 0, "firmware_buffer_drops": 0, - "checked_cases": 23} + "checked_cases": 26} (out / "result.json").write_text(json.dumps(receipt, indent=2) + "\n") print(json.dumps(receipt, indent=2)) diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py index 1439289b1b3247..154c291dda2066 100644 --- a/tools/asahi/g15-zero-words-tests.py +++ b/tools/asahi/g15-zero-words-tests.py @@ -11,7 +11,8 @@ CONFIG_DRM_ASAHI_G15 for the T8122 runtime's UAT adoption (two #[cfg] lines; the tables are unchanged), and when the T6031 board source became the device tree with reference words instead of a board file (the Board enum; -the T6030 tables are unchanged). Its T6030 configuration module is +the T6030 tables are unchanged), and for publication-bound slots (removal of +MAX_VMS and its lifetime-slot comment only). Its T6030 configuration module is independently pinned to the original bytes. prepare_board() is pinned too; it was refreshed for the same Board change (its T6030 arm is unchanged). """ @@ -29,7 +30,7 @@ GOLDEN_FILES = { "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", - "hw.rs": "4d34b69f904c6eb864b7d127763480c01a5e1476a8974184364fa4ba3a90ed63", + "hw.rs": "dd024e27c261635c2edd5356f6d79a6e2072ba57f7d13ada0862491b357cdbf3", "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", } GOLDEN_FUNCTIONS = { diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c index bcf94e6244e690..40e6be5bf24c57 100644 --- a/tools/asahi/t6030-pmp-supplier-tests.c +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -130,6 +130,8 @@ static int pmp_lock, ready_calls, add_calls, add_result; static int poll_calls, device_locked, power_calls, power_result, link_calls; static int find_calls, lockable, present_device, link_ok; static int case_count; +/* The report transaction itself is covered by the G15 PMP-vote selftest. */ +static int gfx_check_result, gfx_check_calls, gfx_set_calls; static struct device_node *of_node_get(struct device_node *node) { @@ -375,6 +377,19 @@ static void queue_work(void *wq, int *work) managed_start.result = ready_result; managed_start.started.done = true; } +static int apple_pmp_report_check_t6030_gfx(struct device *dev) +{ + assert(dev == pmp_device && pmp_lock == 1); + gfx_check_calls++; + return gfx_check_result; +} +static int apple_pmp_report_set_t6030_gfx(struct device *dev, bool enabled) +{ + assert(dev == pmp_device && pmp_lock == 1); + (void)enabled; + gfx_set_calls++; + return 0; +} #include "t6030-pmp-production.h" static void balanced(void) @@ -457,6 +472,7 @@ static void reset(void) pmp_data = (void *)1; ready_calls = add_calls = add_result = poll_calls = 0; power_calls = power_result = link_calls = find_calls = 0; + gfx_check_result = gfx_check_calls = gfx_set_calls = 0; lockable = present_device = link_ok = 1; status_word = PMP_REPORT_READY; } @@ -510,6 +526,9 @@ int main(void) balanced(); case_count++; RUN((void)0, 0, 0, 1); + assert(gfx_check_calls == 1 && gfx_set_calls == 0); + RUN(gfx_check_result = -EINVAL, -EINVAL, 0, 0); + assert(gfx_check_calls == 1 && gfx_set_calls == 0); RUN(report14.child = &entry7; entry7.sibling = &entry17; entry17.sibling = NULL, 0, 0, 1); RUN(apple_pmp_starts.first = NULL, -ENODEV, 0, 0); diff --git a/tools/asahi/t6030-pmp-supplier-tests.py b/tools/asahi/t6030-pmp-supplier-tests.py index d7934acd3ae98c..57b4900843cf43 100755 --- a/tools/asahi/t6030-pmp-supplier-tests.py +++ b/tools/asahi/t6030-pmp-supplier-tests.py @@ -6,7 +6,9 @@ extracted. OF enumeration/property storage, device binding, MMIO polling, and locks are controlled host boundaries. The T6030 managed entry wait and start wait are extracted too; the firmware/workqueue boundary is controlled. -This models ownership/error handling, not real teardown concurrency or DMA. +The GFX-report service is modeled separately from supplier readiness; its +transaction has its own production-body PMP-vote selftest. This models +ownership/error handling, not real teardown concurrency or DMA. """ import argparse import hashlib @@ -146,6 +148,8 @@ def main(): ("gate-config", "IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && gate_soc == &gate_t6030", "gate_soc == &gate_t6030"), ("gate-phandle-shape", 'of_property_count_u32_elems(gpu, "apple,pmp") != 1', "false"), ("serialized-command", "\tguard(mutex)(&pmp_lock);", "\t(void)&pmp_lock;"), + ("gfx-check-error", "ret = apple_pmp_report_check_t6030_gfx(pmp_device);", + "ret = (apple_pmp_report_check_t6030_gfx(pmp_device), 0);"), ] for name, old, new in mutations: assert code.count(old) == 1, (name, code.count(old)) diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c new file mode 100644 index 00000000000000..806148f64d1bff --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/pmp-vote.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Exercise the production PMP vote transaction with failed firmware replies. */ +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +#define CONFIG_APPLE_T6030_GPU_PMP_LINK 1 +#define IS_ENABLED(x) (x) +#define guard(kind) ignore_lock +#define dev_warn(...) ((void)0) +static int pmp_lock; +static void ignore_lock(void *p) { (void)p; } +struct device { void *of_node; }; +static struct device owner; +static struct device *pmp_device = &owner; +static const void *pmp_data = &owner; +static bool t6030, requested; +static int ready, check, power_result[2], report_result[3], powers, reports; +static char calls[16]; +static unsigned int used; +static void called(char c) { assert(used < sizeof(calls) - 1); calls[used++] = c; } +static bool of_machine_is_compatible(const char *s) { (void)s; return t6030; } +static bool of_device_is_compatible(void *p, const char *s) +{ + (void)p; (void)s; return t6030; +} +static int apple_pmp_report_wait_supplier_ready(struct device *d, unsigned int ms) +{ + assert(d == &owner && ms == 5000); called('W'); return ready; +} +static int apple_pmp_send_power_command(const void *p, u8 cmd, u16 id, u32 enabled) +{ + assert(p == &owner && cmd == 0x0f && id == 5); + called(enabled ? 'P' : 'p'); assert(powers < 2); return power_result[powers++]; +} +static int apple_pmp_report_check_t6030_gfx(struct device *d) +{ + assert(d == &owner); called('V'); return check; +} +static int apple_pmp_report_set_t6030_gfx(struct device *d, bool enabled) +{ + assert(d == &owner); called(enabled ? 'R' : 'r'); requested = enabled; + assert(reports < 3); return report_result[reports++]; +} +#include "pmp-vote-production.h" +static void reset(void) +{ + pmp_data = &owner; t6030 = true; requested = false; ready = check = 0; powers = reports = used = 0; + memset(calls, 0, sizeof(calls)); + memset(power_result, 0, sizeof(power_result)); + memset(report_result, 0, sizeof(report_result)); +} +static void result(int expected, const char *sequence, bool enabled) +{ + assert(apple_pmp_set_device_power(0x0f, 5, enabled) == expected); + assert(strcmp(calls, sequence) == 0); +} +int main(void) +{ + reset(); + assert(apple_pmp_set_device_power(0, 5, 1) == -EINVAL); + assert(apple_pmp_set_device_power(0x0f, 16, 1) == -EINVAL); + assert(apple_pmp_set_device_power(0x0f, 5, 2) == -EINVAL); + assert(!used); + pmp_data = NULL; result(-ENODEV, "", true); + reset(); ready = -EAGAIN; result(-EAGAIN, "W", true); + reset(); t6030 = false; result(0, "WP", true); + reset(); t6030 = false; result(0, "Wp", false); + reset(); check = -EINVAL; result(-EINVAL, "WV", true); + reset(); result(0, "WVP", true); + /* Power-on must not start bandwidth tracking before firmware READY. */ + assert(!reports); + assert(apple_pmp_t6030_gpu_ready() == 0); + assert(strcmp(calls, "WVPrR") == 0 && requested); + reset(); requested = true; + assert(apple_pmp_t6030_gpu_ready() == 0); + assert(strcmp(calls, "rR") == 0 && requested); + reset(); pmp_data = NULL; + assert(apple_pmp_t6030_gpu_ready() == -ENODEV && !used); + reset(); report_result[0] = -ETIMEDOUT; + assert(apple_pmp_t6030_gpu_ready() == -ETIMEDOUT); + assert(strcmp(calls, "r") == 0); + reset(); report_result[1] = -ETIMEDOUT; + assert(apple_pmp_t6030_gpu_ready() == -ETIMEDOUT); + assert(strcmp(calls, "rR") == 0); + reset(); result(0, "WVrp", false); + reset(); power_result[0] = -ETIMEDOUT; result(-ETIMEDOUT, "WVP", true); + reset(); report_result[0] = -ETIMEDOUT; result(-ETIMEDOUT, "WVrR", false); + reset(); report_result[0] = -ETIMEDOUT; report_result[1] = -EIO; + result(-ETIMEDOUT, "WVrR", false); + reset(); power_result[0] = -EIO; result(-EIO, "WVrpR", false); + puts("production PMP vote: scope, sequencing, reply errors and rollback: PASS"); +} diff --git a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py index be10389b611ed5..202830cd8d716e 100644 --- a/tools/testing/selftests/drivers/gpu/asahi-g15/run.py +++ b/tools/testing/selftests/drivers/gpu/asahi-g15/run.py @@ -114,6 +114,7 @@ pub const UAT_PGSZ: usize = 0x4000; pub const UAT_PGMSK: usize = UAT_PGSZ - 1; pub struct Vm(pub Vec); + pub type VmReadGuard = Vm; impl Vm { pub fn read_user(&self, va: u64, out: &mut [u8]) -> Result { let start = usize::try_from(va).map_err(|_| EFAULT)?; @@ -133,7 +134,7 @@ /// cover() calls: the page checks the walk made. pub static COVERS: Cell = Cell::new(0); } - pub fn cover(vm: &mmu::Vm, start: u64, end: u64, _: bool, _: bool) -> Result { + pub fn cover(vm: &mut mmu::VmReadGuard, start: u64, end: u64, _: bool, _: bool) -> Result { COVERS.with(|c| c.set(c.get() + 1)); if start >= SYNTHETIC { return Ok(()); @@ -156,34 +157,34 @@ } #[test] fn terminator_and_known_barrier() { - assert_eq!(gate::cdm(&vm(&[0x40000000]), 0, 0), Ok(())); - assert_eq!(gate::cdm(&vm(&[0x60000160, 0x40000000]), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&[0x40000000]), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&[0x60000160, 0x40000000]), 0, 0), Ok(())); } #[test] fn cyclic_links_are_rejected() { - assert_eq!(gate::cdm(&vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&[0x20000000, 0]), 0, 0), Err(EINVAL)); } /// Words of 4 bytes that fill the byte budget: the root block and one per link. const BUDGET_WORDS: usize = 65 * 0x10000 / 4; #[test] fn byte_budget_requires_a_terminator() { - assert_eq!(gate::cdm(&vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&vec![0x60000160; BUDGET_WORDS + 1]), 0, 0), Err(EINVAL)); } #[test] fn final_record_may_terminate() { let mut words = vec![0x60000160; BUDGET_WORDS + 1]; words[BUDGET_WORDS - 1] = 0x40000000; - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); words[BUDGET_WORDS - 1] = 0x60000160; words[BUDGET_WORDS] = 0x40000000; - assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Err(EINVAL)); } #[test] fn more_than_8192_records_are_admitted() { // 1024 draws of ten records each, as a tessellated multi-draw writes them. let mut words = vec![0x60000160; 10240]; words.push(0x40000000); - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); } #[test] fn link_budget_still_bounds_the_walk() { @@ -193,13 +194,13 @@ words.extend([0x20000000, (i + 1) * 8]); } words.push(0x40000000); - assert_eq!(gate::cdm(&vm(&words), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Err(EINVAL)); let mut words = Vec::new(); for i in 0..64u32 { words.extend([0x20000000, (i + 1) * 8]); } words.push(0x40000000); - assert_eq!(gate::cdm(&vm(&words), 0, 0), Ok(())); + assert_eq!(gate::cdm(&mut vm(&words), 0, 0), Ok(())); } /// The USC window of these cases and its first program, both in the mock's mapped range. const USC: u64 = crate::command::SYNTHETIC; @@ -216,7 +217,7 @@ } fn covers(words: &[u32]) -> (Result, u32) { crate::command::COVERS.with(|c| c.set(0)); - let r = gate::cdm(&vm(words), USC, 0); + let r = gate::cdm(&mut vm(words), USC, 0); (r, crate::command::COVERS.with(|c| c.get())) } const PAGE: u64 = mmu::UAT_PGSZ as u64; @@ -247,9 +248,9 @@ } #[test] fn malformed_and_unreadable_streams_are_rejected() { - assert_eq!(gate::cdm(&vm(&[0xffffffff]), 0, 0), Err(EINVAL)); - assert_eq!(gate::cdm(&vm(&[0x20000000, 2]), 0, 0), Err(EINVAL)); - assert_eq!(gate::cdm(&mmu::Vm(vec![]), 0, 0), Err(EFAULT)); + assert_eq!(gate::cdm(&mut vm(&[0xffffffff]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut vm(&[0x20000000, 2]), 0, 0), Err(EINVAL)); + assert_eq!(gate::cdm(&mut mmu::Vm(vec![]), 0, 0), Err(EFAULT)); } } ''' @@ -299,10 +300,40 @@ def main(): str(Path(__file__).with_name("pmp-bridge.c")), "-o", str(output)], check=True) subprocess.run([str(output)], check=True) + owner = (ROOT / "drivers/soc/apple/pmp-export.c").read_text() + start = owner.index("int apple_pmp_set_device_power(") + end = owner.index("\n}", start) + 2 + vote = owner[start:end] + start = owner.index("int apple_pmp_t6030_gpu_ready(") + end = owner.index("\n}", start) + 2 + (temp / "pmp-vote-production.h").write_text(vote + "\n" + owner[start:end]) + output = temp / "pmp-vote" + subprocess.run([os.environ.get("HOSTCC", "cc"), "-std=c11", "-Wall", "-Wextra", + "-Werror", "-I", str(temp), + str(Path(__file__).with_name("pmp-vote.c")), "-o", str(output)], check=True) + subprocess.run([str(output)], check=True) + # Native wire layout is a private per-backend facade. Its independent host test # checks that canonical normal/G17 UAPI remains unchanged; there is no global # Rust bindgen header selector in this integration. harness = RUST_PREFIX + f'\n#[path = "{G15 / "gate.rs"}"] mod gate;\n' + encode = (G15 / "encode.rs").read_text() + start = encode.index("pub(crate) const fn stamp_seed(") + end = encode.index("\n}", start) + 2 + harness += encode[start:end] + ''' +#[test] +fn firmware_stamp_wrap() { + for seed in [0xc1000000u32, 0x7a000000, 0x3d000000] { + let wrap = ((1u64 << 32) - seed as u64) / 0x100; + assert_eq!(stamp_seed(seed, 0), seed); + assert_eq!(stamp_seed(seed, wrap - 1), 0xffffff00); + assert_eq!(stamp_seed(seed, wrap), 0); + assert_eq!(stamp_seed(seed, wrap + 1), 0x100); + assert_eq!(stamp_seed(seed, 1 << 24), seed); + assert_eq!(stamp_seed(seed, u64::MAX), seed.wrapping_sub(0x100)); + } +} +''' harness += f'#[path = "{G15 / "recovery/logic.rs"}"] mod recovery;\n' harness += f'#[path = "{G15 / "power/idle.rs"}"] mod idle;\n' harness += f'#[path = "{G15 / "power/pstate.rs"}"] mod pstate;\n'