From 84ef8e4d724d2086fea6172994d3d29a9c8c2666 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Thu, 22 Oct 2026 23:16:35 -0400 Subject: [PATCH 01/37] drm/asahi: m3: restore the 7.1.12 T8122 runtime sources, unwired Bring back, unchanged, the M3 runtime that starts the T8122 (M3 MacBook Air) GPU on the release-20 kernel (c46d7dcdb871, the source of linux-aurora-m3gpu 7.1.12.aurora2-20261017.20): the runtime, its firmware admission, InitData, device, RTKit, memory, submission and client files, the T8122 start experiment, and the 7.1.12 hw tables (hw.rs, hw/t8122.rs, hw/agx3.rs, identity.rs). g15_raw.rs and initdata.rs are the G15 (G15, V14_8_3) expansions of 7.1.12's fw/initdata.rs and initdata.rs: every #[versions(AGX)] item replaced by its G15V14_8_3 instance, which 7.2's version matrix no longer generates. The expansion keeps the source text; its eleven G15 structures match rustc's own expansion of c46d7dcdb871 field for field. Nothing declares these modules yet, so this changes no build. The next commits adapt them to the 7.2 driver. Co-authored-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-authored-by: Eryk Wieliczko Co-authored-by: Satya Benson Co-authored-by: Naeem Malik Co-authored-by: garymeyerca Co-authored-by: DjDeveloperr Generated-by: LLM Co-Authored-By: Claude Opus 5.5 Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/agx_attachments.rs | 71 + drivers/gpu/drm/asahi/m3/agx_compute.rs | 9 + drivers/gpu/drm/asahi/m3/agx_fault.rs | 87 + drivers/gpu/drm/asahi/m3/agx_memory.rs | 12 + drivers/gpu/drm/asahi/m3/agx_resources.rs | 170 ++ drivers/gpu/drm/asahi/m3/agx_status.rs | 34 + drivers/gpu/drm/asahi/m3/agx_uapi.rs | 721 ++++++++ drivers/gpu/drm/asahi/m3/g15_raw.rs | 1800 +++++++++++++++++++ drivers/gpu/drm/asahi/m3/hw.rs | 679 +++++++ drivers/gpu/drm/asahi/m3/hw/agx3.rs | 59 + drivers/gpu/drm/asahi/m3/hw/t8122.rs | 147 ++ drivers/gpu/drm/asahi/m3/identity.rs | 67 + drivers/gpu/drm/asahi/m3/initdata.rs | 1457 +++++++++++++++ drivers/gpu/drm/asahi/m3/m3_adt_config.rs | 1491 +++++++++++++++ drivers/gpu/drm/asahi/m3/m3_board.rs | 559 ++++++ drivers/gpu/drm/asahi/m3/m3_client.rs | 228 +++ drivers/gpu/drm/asahi/m3/m3_completion.rs | 66 + drivers/gpu/drm/asahi/m3/m3_compute.rs | 379 ++++ drivers/gpu/drm/asahi/m3/m3_config.rs | 444 +++++ drivers/gpu/drm/asahi/m3/m3_device.rs | 315 ++++ drivers/gpu/drm/asahi/m3/m3_drm.rs | 210 +++ drivers/gpu/drm/asahi/m3/m3_firmware.rs | 441 +++++ drivers/gpu/drm/asahi/m3/m3_memory.rs | 163 ++ drivers/gpu/drm/asahi/m3/m3_params.rs | 900 ++++++++++ drivers/gpu/drm/asahi/m3/m3_pass.rs | 51 + drivers/gpu/drm/asahi/m3/m3_pool.rs | 41 + drivers/gpu/drm/asahi/m3/m3_render.rs | 696 +++++++ drivers/gpu/drm/asahi/m3/m3_resources.rs | 51 + drivers/gpu/drm/asahi/m3/m3_rtkit.rs | 327 ++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 1246 +++++++++++++ drivers/gpu/drm/asahi/m3/m3_submit.rs | 297 +++ drivers/gpu/drm/asahi/m3/m3_thermal.rs | 499 +++++ drivers/gpu/drm/asahi/m3/t8122_knobs.rs | 333 ++++ drivers/gpu/drm/asahi/m3/t8122_start.rs | 579 ++++++ 34 files changed, 14629 insertions(+) create mode 100644 drivers/gpu/drm/asahi/m3/agx_attachments.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_compute.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_fault.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_memory.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_resources.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_status.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_uapi.rs create mode 100644 drivers/gpu/drm/asahi/m3/g15_raw.rs create mode 100644 drivers/gpu/drm/asahi/m3/hw.rs create mode 100644 drivers/gpu/drm/asahi/m3/hw/agx3.rs create mode 100644 drivers/gpu/drm/asahi/m3/hw/t8122.rs create mode 100644 drivers/gpu/drm/asahi/m3/identity.rs create mode 100644 drivers/gpu/drm/asahi/m3/initdata.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_adt_config.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_board.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_client.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_completion.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_compute.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_config.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_device.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_drm.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_firmware.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_memory.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_params.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_pass.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_pool.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_render.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_resources.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_rtkit.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_runtime.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_submit.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_thermal.rs create mode 100644 drivers/gpu/drm/asahi/m3/t8122_knobs.rs create mode 100644 drivers/gpu/drm/asahi/m3/t8122_start.rs diff --git a/drivers/gpu/drm/asahi/m3/agx_attachments.rs b/drivers/gpu/drm/asahi/m3/agx_attachments.rs new file mode 100644 index 0000000000000..5aeb3c7335c22 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_attachments.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +pub(crate) const SIZE: usize = 0x104; +pub(crate) const CAPACITY: usize = 16; + +#[derive(Clone, Copy)] +pub(crate) struct Attachments(pub(crate) [u8; SIZE]); + +impl Attachments { + pub(crate) const EMPTY: Self = Self([0; SIZE]); + pub(crate) fn count(&self) -> u8 { self.0[0x100] } + + /// Normalize byte ranges to the firmware's 128-byte cache-line units. + /// The caller must validate the entire normalized writable GPU range. + pub(crate) fn range(address: u64, size: u64) -> Result<(u64, u64), ()> { + if address == 0 || size == 0 { return Err(()); } + let end = address.checked_add(size).and_then(|v| v.checked_add(127)).ok_or(())? & !127; + let start = address & !127; + if start == 0 || end > 1 << 42 || (end - start) >> 7 > u32::MAX as u64 { + return Err(()); + } + Ok((start, end - start)) + } + + pub(crate) fn new(ranges: &[(u64, u64)]) -> Result { + if ranges.len() > CAPACITY { return Err(()); } + let mut out = Self::EMPTY; + for (record, &(address, size)) in out.0[..0x100].chunks_exact_mut(16).zip(ranges) { + let (address, size) = Self::range(address, size)?; + record[..8].copy_from_slice(&address.to_le_bytes()); + record[8..12].copy_from_slice(&((size >> 7) as u32).to_le_bytes()); + record[12..14].copy_from_slice(&0x18u16.to_le_bytes()); + record[14..16].copy_from_slice(&1u16.to_le_bytes()); + } + out.0[0x100..].copy_from_slice(&(ranges.len() as u32).to_le_bytes()); + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn covers_both_partial_edge_lines() { + let a = Attachments::new(&[(0x100007f, 2)]).unwrap(); + assert_eq!(u64::from_le_bytes(a.0[..8].try_into().unwrap()), 0x1000000); + assert_eq!(u32::from_le_bytes(a.0[8..12].try_into().unwrap()), 2); + assert_eq!(&a.0[12..16], &[0x18, 0, 1, 0]); + assert_eq!(a.count(), 1); + assert!(a.0[16..0x100].iter().all(|v| *v == 0)); + } + #[test] + fn rejects_unrepresentable_ranges_and_count() { + for range in [(0, 128), (128, 0), (u64::MAX - 1, 8), + ((1 << 42) - 128, 129), (128, (u32::MAX as u64 + 1) << 7)] { + assert!(Attachments::new(&[range]).is_err()); + } + assert!(Attachments::new(&[(128, 128); CAPACITY + 1]).is_err()); + let full = Attachments::new(&[(128, 128); CAPACITY]).unwrap(); + assert_eq!(full.count(), 16); + assert_eq!(&full.0[0x100..], &[16, 0, 0, 0]); + assert_eq!(Attachments::new(&[]).unwrap().0, Attachments::EMPTY.0); + } + #[test] + fn matches_native_m4_writable_buffer_descriptor() { + let encoded = Attachments::new(&[(0x100000a8000, 0x400000)]).unwrap(); + assert_eq!(&encoded.0[..16], &[ + 0, 0x80, 0x0a, 0, 0, 1, 0, 0, 0, 0x80, 0, 0, 0x18, 0, 1, 0]); + assert_eq!(encoded.count(), 1); + } +} diff --git a/drivers/gpu/drm/asahi/m3/agx_compute.rs b/drivers/gpu/drm/asahi/m3/agx_compute.rs new file mode 100644 index 0000000000000..d58005c504f7a --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_compute.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +/// Complete client CDM range and its USC address window. +#[derive(Clone, Copy)] +pub(crate) struct Control { + pub base: u64, + pub end: u64, + pub usc_base: u64, +} diff --git a/drivers/gpu/drm/asahi/m3/agx_fault.rs b/drivers/gpu/drm/asahi/m3/agx_fault.rs new file mode 100644 index 0000000000000..9c37cfa98dce9 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_fault.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Immutable fault-only snapshots of owned M3 firmware memory. +//! +//! Records contain name[32], firmware VA, length, begin/end monotonic ns, +//! then exactly `length` bytes. No MMIO or application BOs are read. Reads +//! are sequential observations, not atomic with running firmware. + +use kernel::{prelude::*, time::{ClockSource, Monotonic}}; + +pub(crate) struct Dump { + bytes: KVVec, + count: u32, + limit: usize, +} + +impl Dump { + + /// M3 snapshots have no scheduler job ID yet and contain only bounded + /// driver-owned firmware configuration and channel storage. + pub(crate) fn new_m3() -> Result { + Self::with_format(0, b"M3FWD001", 2 * 1024 * 1024) + } + + fn with_format(job: u64, magic: &[u8; 8], limit: usize) -> Result { + // Reserve before firmware/job publication, while healthy. Record appends + // must never enter reclaim waiting on the GPU that just faulted. + let mut bytes = KVVec::with_capacity(limit, GFP_KERNEL)?; + bytes.resize(40, 0, GFP_KERNEL)?; + bytes[..8].copy_from_slice(magic); + bytes[16..24].copy_from_slice(&job.to_le_bytes()); + bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); + Ok(Self { bytes, count: 0, limit }) + } + + /// Start the one-shot capture; construction may precede the fault by hours. + pub(crate) fn begin(&mut self, job: u64) { + self.bytes[16..24].copy_from_slice(&job.to_le_bytes()); + self.bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); + } + + pub(crate) fn record(&mut self, name: &str, va: u64, size: usize, + read: impl FnOnce(&mut [u8]) -> Result) -> Result { + if name.is_empty() || name.len() >= 32 || size == 0 { return Err(EINVAL); } + let count = self.count.checked_add(1).ok_or(EOVERFLOW)?; + let start = self.bytes.len(); + let end = start.checked_add(64).and_then(|x| x.checked_add(size)).ok_or(EOVERFLOW)?; + if end > self.limit { return Err(E2BIG); } + // Capacity was reserved at construction. NOWAIT is defense in depth; + // the checked limit above guarantees resize cannot allocate here. + self.bytes.resize(end, 0, GFP_NOWAIT)?; + self.bytes[start..start+name.len()].copy_from_slice(name.as_bytes()); + self.bytes[start+32..start+40].copy_from_slice(&va.to_le_bytes()); + self.bytes[start+40..start+48].copy_from_slice(&(size as u64).to_le_bytes()); + let begin = Monotonic::ktime_get() as u64; + if let Err(error) = read(&mut self.bytes[start+64..end]) { + self.bytes.truncate(start); + return Err(error); + } + let finish = Monotonic::ktime_get() as u64; + self.bytes[start+48..start+56].copy_from_slice(&begin.to_le_bytes()); + self.bytes[start+56..start+64].copy_from_slice(&finish.to_le_bytes()); + self.count = count; + Ok(()) + } + + pub(crate) fn publish(mut self, dev: &kernel::device::Device) -> Result { + let length = self.bytes.len(); + self.bytes[8..12].copy_from_slice(&self.count.to_le_bytes()); + self.bytes[12..16].copy_from_slice(&1u32.to_le_bytes()); + self.bytes[32..40].copy_from_slice(&(length as u64).to_le_bytes()); + let format = "M3FWD001"; + let owned = KBox::new(self, GFP_NOWAIT)?; + kernel::devcoredump::dev_coredump(dev, &crate::THIS_MODULE, owned, GFP_NOWAIT, + kernel::devcoredump::DEFAULT_TIMEOUT); + dev_info!(dev, "Asahi: sealed {}-byte firmware fault snapshot offered to devcoredump ({})\n", length, format); + Ok(()) + } +} + +impl kernel::devcoredump::DevCoreDump for Dump { + fn read(&self, output: &mut [u8], offset: usize) -> Result { + if offset >= self.bytes.len() { return Ok(0); } + let length = output.len().min(self.bytes.len()-offset); + output[..length].copy_from_slice(&self.bytes[offset..offset+length]); + Ok(length) + } +} diff --git a/drivers/gpu/drm/asahi/m3/agx_memory.rs b/drivers/gpu/drm/asahi/m3/agx_memory.rs new file mode 100644 index 0000000000000..521990b4aad21 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_memory.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +// Firmware validates Timestamp user destinations against a dedicated 64 MiB +// interval published in HwDataB+0x28. Reserve the last 64 MiB of the +// firmware VM selector, disjoint from MMIO/command allocations above. +pub(crate) const TIMESTAMP_RANGE: core::ops::Range = + 0xffff_fc2f_fc00_0000..0xffff_fc30_0000_0000; + +pub(crate) fn publish() { + // SAFETY: Orders preceding WC buffer stores before a device doorbell. + unsafe { core::arch::asm!("dsb oshst", options(nostack, preserves_flags)) }; +} diff --git a/drivers/gpu/drm/asahi/m3/agx_resources.rs b/drivers/gpu/drm/asahi/m3/agx_resources.rs new file mode 100644 index 0000000000000..8fdf87bdb046d --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_resources.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! J713 bootloader resource handoff. Reading this metadata does not map memory, +//! claim UAT ownership, authenticate firmware, or grant permission to start it. +//! Firmware ABI admission and an owned runtime are separate steps. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Region { + pub(crate) base: u64, + pub(crate) size: u64, +} + +impl Region { + fn end(self) -> Option { + self.base.checked_add(self.size) + } + + fn valid(self) -> bool { + self.base != 0 + && self.size != 0 + && (self.base | self.size) & 0x3fff == 0 + && self.end().is_some_and(|end| end <= 1 << 42) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ResourceError { + Version, + Region, + Overlap, + FirmwareLayout, +} + +#[derive(Debug)] +pub(crate) struct Resources { + /// ttbs, pagetables, handoff, shared-l2, fw-text, fw-data. + pub(crate) regions: [Region; 6], + pub(crate) firmware_vas: [u64; 2], +} + +impl Resources { + pub(crate) fn validate( + version: u32, + regions: [Region; 6], + firmware_vas: [u64; 2], + firmware_flags: [u32; 2], + ) -> Result { + if version != 1 { + return Err(ResourceError::Version); + } + for (i, region) in regions.iter().enumerate() { + if !region.valid() { + return Err(ResourceError::Region); + } + for other in ®ions[..i] { + if region.base < other.end().unwrap() && other.base < region.end().unwrap() { + return Err(ResourceError::Overlap); + } + } + } + // The inherited 42-bit page-table roots and all 64 16-byte TTBAT + // entries fit their reserved regions. No panel-sized allocation pool. + if firmware_flags != [1, 0] + || firmware_vas[0] != 0xffff_fc00_0000_0000 + || firmware_vas[0].checked_add(regions[4].size) != Some(firmware_vas[1]) + || firmware_vas[1].checked_add(regions[5].size).is_none() + { + return Err(ResourceError::FirmwareLayout); + } + Ok(Self { + regions, + firmware_vas, + }) + } +} + + + +#[cfg(test)] +mod tests { + use super::*; + + fn observed() -> [Region; 6] { + [ + (0x103fffb8000, 0x4000), + (0x103fff78000, 0x40000), + (0x103fff70000, 0x4000), + (0x103fff74000, 0x4000), + (0x10000cc0000, 0x60000), + (0x10001df4000, 0x13c000), + ] + .map(|(base, size)| Region { base, size }) + } + const VAS: [u64; 2] = [0xfffffc0000000000, 0xfffffc0000060000]; + + #[test] + fn observed_and_relocated_boot_addresses() { + let res = Resources::validate(1, observed(), VAS, [1, 0]).unwrap(); + assert_eq!(res.firmware_vas, VAS); + assert_eq!(res.regions[2].base, 0x103fff70000); + let relocated = observed().map(|r| Region { + base: r.base + 0x400000000, + ..r + }); + assert!(Resources::validate(1, relocated, VAS, [1, 0]).is_ok()); + } + + /// Relocating the UAT reservations preserves their sizes and ownership checks. + #[test] + fn relocated_uat_regions_keep_firmware_segments() { + let mut regions = observed(); + for region in &mut regions[..4] { + region.base -= 0x40000; + } + let res = Resources::validate(1, regions, VAS, [1, 0]).unwrap(); + assert_eq!(res.regions[0].base, 0x103fff78000); + assert_eq!(res.regions[4], observed()[4]); + assert_eq!(res.regions[5], observed()[5]); + } + + #[test] + fn rejects_missing_overlapping_unaligned_and_out_of_range_memory() { + for r in [ + Region { base: 0, size: 0 }, + Region { + base: 0x12345, + size: 0x4000, + }, + Region { + base: 1 << 42, + size: 0x4000, + }, + Region { + base: u64::MAX & !0x3fff, + size: 0x4000, + }, + ] { + let mut bad = observed(); + bad[0] = r; + assert!(matches!( + Resources::validate(1, bad, VAS, [1, 0]), + Err(ResourceError::Region) + )); + } + let mut bad = observed(); + bad[0] = bad[1]; + assert!(matches!( + Resources::validate(1, bad, VAS, [1, 0]), + Err(ResourceError::Overlap) + )); + } + + #[test] + fn rejects_unknown_handoff_and_segment_layouts() { + assert!(matches!( + Resources::validate(0, observed(), VAS, [1, 0]), + Err(ResourceError::Version) + )); + for (vas, flags) in [ + (VAS, [0, 1]), + ([VAS[0], VAS[1] + 0x4000], [1, 0]), + ([0xfffffe0000000000, VAS[1]], [1, 0]), + ] { + assert!(matches!( + Resources::validate(1, observed(), vas, flags), + Err(ResourceError::FirmwareLayout) + )); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/agx_status.rs b/drivers/gpu/drm/asahi/m3/agx_status.rs new file mode 100644 index 0000000000000..1d83086a1d155 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_status.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! CPU-owned sticky VM failure reporting. No GPU mapping or firmware access. +use core::sync::atomic::{AtomicI32, Ordering}; + +pub(crate) const PARAM_GROUP_VM_STATUS: u32 = 1; +pub(crate) const FEATURE_VM_STATUS: u64 = 1 << 1; +pub(crate) const VM_STATUS_SIZE: usize = 16; + +/// Shared by the VM and accepted jobs; never cleared by queue destruction or +/// successful work. A newly created VM receives a new status object. +pub(crate) struct VmStatus { error: AtomicI32 } +impl VmStatus { + pub(crate) fn new() -> Self { Self { error: AtomicI32::new(0) } } + pub(crate) fn record(&self, error: i32) { + if error < 0 { + let _ = self.error.compare_exchange(0, error, Ordering::AcqRel, Ordering::Acquire); + } + } + pub(crate) fn get(&self) -> i32 { self.error.load(Ordering::Acquire) } +} + +/// GET_PARAMS group 1 is an in/out request: vm_id, flags=0, error(out), pad=0. +/// Decode once, before lookup, so a racing userspace edit cannot change owner. +pub(crate) fn decode_request(bytes: &[u8; VM_STATUS_SIZE]) -> Option { + let word = |i| u32::from_ne_bytes(bytes[i..i+4].try_into().unwrap()); + if word(4) != 0 || word(12) != 0 { return None; } + Some(word(0)) +} +pub(crate) fn encode_response(vm_id: u32, error: i32) -> [u8; VM_STATUS_SIZE] { + let mut bytes = [0; VM_STATUS_SIZE]; + bytes[0..4].copy_from_slice(&vm_id.to_ne_bytes()); + bytes[8..12].copy_from_slice(&error.to_ne_bytes()); + bytes +} diff --git a/drivers/gpu/drm/asahi/m3/agx_uapi.rs b/drivers/gpu/drm/asahi/m3/agx_uapi.rs new file mode 100644 index 0000000000000..60107f8833edc --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_uapi.rs @@ -0,0 +1,721 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Shared modern-Asahi UAPI parsing. +//! +//! The byte layouts match `drm_asahi_cmd_header`, `drm_asahi_cmd_render`, +//! `drm_asahi_cmd_compute`, and `drm_asahi_attachment`. Translation validates +//! the queue's 4 GiB USC window and every userspace range before constructing +//! M3 render parameters or compute-entry operands. This module allocates no +//! object, publishes no queue state, and performs no firmware or MMIO action. + +pub(crate) const UAPI_COMMAND_HEADER_SIZE: usize = 8; +pub(crate) const UAPI_RENDER_SIZE: usize = 256; +pub(crate) const UAPI_COMPUTE_SIZE: usize = 64; +pub(crate) const UAPI_ATTACHMENT_SIZE: usize = 24; +pub(crate) const UAPI_MAX_ATTACHMENTS: usize = 16; +pub(crate) const UAPI_MAX_HARDWARE_COMMANDS: u32 = 64; +pub(crate) const UAPI_BARRIER_NONE: u16 = 0xffff; +pub(crate) const USC_WINDOW_SIZE: u64 = 1 << 32; +const COMMAND_RENDER: u16 = 0; +const COMMAND_COMPUTE: u16 = 1; +const SET_VERTEX_ATTACHMENTS: u16 = 2; +const SET_FRAGMENT_ATTACHMENTS: u16 = 3; +const SET_COMPUTE_ATTACHMENTS: u16 = 4; + +const RENDER_VERTEX_SCRATCH: u32 = 1 << 0; +const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1; +const RENDER_NO_VERTEX_CLUSTERING: u32 = 1 << 2; +const RENDER_RSRC_SPEC_HI: u32 = 1 << 4; +const RENDER_DBIAS_IS_INT: u32 = 1 << 18; +const RENDER_FLAGS: u32 = RENDER_VERTEX_SCRATCH + | RENDER_PROCESS_EMPTY_TILES + | RENDER_NO_VERTEX_CLUSTERING + | RENDER_RSRC_SPEC_HI + | RENDER_DBIAS_IS_INT; + +const SAMPLER_SIZE: u64 = 8; +const MAX_SAMPLERS: u32 = 1024; +const PAGE_SIZE: u64 = 0x4000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum UapiTranslateError { + HeaderTruncated, + PayloadTruncated, + PayloadTail, + CommandType, + AttachmentPayloadSize, + AttachmentCount, + AttachmentFields, + AttachmentBarrier, + HardwareCommandCount, + BarrierFuture, + QueueUscAlignment, + QueueUscRange, + AddressRange, + AddressAlignment, + FlagsUnknownBits, + FlagsUnsupported, + Dimensions, + Layers, + UtileDimensions, + Samples, + TilebufferSize, + SamplerCount, + SamplerPair, + HelperProgram, + Add3ProofAttachments, + ProgramOffset, + ZlsFields, + StencilClearBits, +} + +impl UapiTranslateError { + /// Stable short name for one rejection, so a failing submit reports which + /// check refused it instead of a bare EINVAL. Kept as `&'static str` (no + /// `Debug` formatting) to stay cheap on the ioctl path. + pub(crate) fn name(&self) -> &'static str { + match self { + Self::HeaderTruncated => "header-truncated", + Self::PayloadTruncated => "payload-truncated", + Self::PayloadTail => "payload-tail", + Self::CommandType => "command-type", + Self::AttachmentPayloadSize => "attachment-payload-size", + Self::AttachmentCount => "attachment-count", + Self::AttachmentFields => "attachment-fields", + Self::AttachmentBarrier => "attachment-barrier", + Self::HardwareCommandCount => "hardware-command-count", + Self::BarrierFuture => "barrier-future", + Self::QueueUscAlignment => "queue-usc-alignment", + Self::QueueUscRange => "queue-usc-range", + Self::AddressRange => "address-range", + Self::AddressAlignment => "address-alignment", + Self::FlagsUnknownBits => "flags-unknown-bits", + Self::FlagsUnsupported => "flags-unsupported", + Self::Dimensions => "dimensions", + Self::Layers => "layers", + Self::UtileDimensions => "utile-dimensions", + Self::Samples => "samples", + Self::TilebufferSize => "tilebuffer-size", + Self::SamplerCount => "sampler-count", + Self::SamplerPair => "sampler-pair", + Self::HelperProgram => "helper-program", + Self::Add3ProofAttachments => "add3-proof-attachments", + Self::ProgramOffset => "program-offset", + Self::ZlsFields => "zls-fields", + Self::StencilClearBits => "stencil-clear-bits", + + + } + } +} + +fn get_u16(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) +} + +fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) +} + +fn padded_payload(raw: &[u8]) -> Result<[u8; N], UapiTranslateError> { + if raw.len() > N && raw[N..].iter().any(|byte| *byte != 0) { + return Err(UapiTranslateError::PayloadTail); + } + let mut out = [0u8; N]; + let count = core::cmp::min(raw.len(), N); + out[..count].copy_from_slice(&raw[..count]); + Ok(out) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiCommandHeader { + pub(crate) command_type: u16, + pub(crate) size: u16, + pub(crate) render_barrier: u16, + pub(crate) compute_barrier: u16, +} + +impl UapiCommandHeader { + fn parse(raw: &[u8]) -> Result { + if raw.len() < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + Ok(Self { + command_type: get_u16(raw, 0), + size: get_u16(raw, 2), + render_barrier: get_u16(raw, 4), + compute_barrier: get_u16(raw, 6), + }) + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct UapiAttachment { + pub(crate) address: u64, + pub(crate) size: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiAttachmentList { + pub(crate) entries: [UapiAttachment; UAPI_MAX_ATTACHMENTS], + pub(crate) count: u8, +} + +impl UapiAttachmentList { + pub(crate) const EMPTY: Self = Self { + entries: [UapiAttachment { + address: 0, + size: 0, + }; UAPI_MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[UapiAttachment] { + &self.entries[..self.count as usize] + } + + fn parse(raw: &[u8]) -> Result { + if raw.len() % UAPI_ATTACHMENT_SIZE != 0 { + return Err(UapiTranslateError::AttachmentPayloadSize); + } + let count = raw.len() / UAPI_ATTACHMENT_SIZE; + if count > UAPI_MAX_ATTACHMENTS { + return Err(UapiTranslateError::AttachmentCount); + } + let mut out = Self::EMPTY; + for index in 0..count { + let offset = index * UAPI_ATTACHMENT_SIZE; + if get_u32(raw, offset + 0x10) != 0 || get_u32(raw, offset + 0x14) != 0 { + return Err(UapiTranslateError::AttachmentFields); + } + out.entries[index] = UapiAttachment { + address: get_u64(raw, offset), + size: get_u64(raw, offset + 8), + }; + } + out.count = count as u8; + Ok(out) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiHelperProgram { + pub(crate) binary: u32, + pub(crate) config: u32, + pub(crate) data: u64, +} + +fn helper(raw: &[u8], offset: usize) -> UapiHelperProgram { + UapiHelperProgram { + binary: get_u32(raw, offset), + config: get_u32(raw, offset + 4), + data: get_u64(raw, offset + 8), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiProgram { + pub(crate) usc: u32, + pub(crate) resource_spec: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamp { + pub(crate) handle: u32, + pub(crate) offset: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamps { + pub(crate) start: UapiTimestamp, + pub(crate) end: UapiTimestamp, +} + +fn timestamps(raw: &[u8], offset: usize) -> UapiTimestamps { + UapiTimestamps { + start: UapiTimestamp { + handle: get_u32(raw, offset), + offset: get_u32(raw, offset + 4), + }, + end: UapiTimestamp { + handle: get_u32(raw, offset + 8), + offset: get_u32(raw, offset + 12), + }, + } +} + +fn program(raw: &[u8], offset: usize, high_offset: usize) -> UapiProgram { + UapiProgram { + usc: get_u32(raw, offset), + resource_spec: get_u32(raw, offset + 4) as u64 + | (get_u32(raw, high_offset) as u64) << 32, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiZlsBuffer { + pub(crate) base: u64, + pub(crate) compression_base: u64, + pub(crate) stride: u32, + pub(crate) compression_stride: u32, +} + +fn zls(raw: &[u8], offset: usize) -> UapiZlsBuffer { + UapiZlsBuffer { + base: get_u64(raw, offset), + compression_base: get_u64(raw, offset + 8), + stride: get_u32(raw, offset + 0x10), + compression_stride: get_u32(raw, offset + 0x14), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiRenderCommand { + pub(crate) flags: u32, + pub(crate) depth_dimensions: u32, + pub(crate) vdm_base: u64, + pub(crate) vertex_helper: UapiHelperProgram, + pub(crate) fragment_helper: UapiHelperProgram, + pub(crate) scissor_base: u64, + pub(crate) depth_bias_base: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth: UapiZlsBuffer, + pub(crate) stencil: UapiZlsBuffer, + pub(crate) zls_control: u64, + pub(crate) multisample_control: u64, + pub(crate) sampler_heap: u64, + pub(crate) ppp_control: u32, + pub(crate) width: u16, + pub(crate) height: u16, + pub(crate) layers: u16, + pub(crate) sampler_count: u16, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) samples: u8, + pub(crate) sample_size: u8, + pub(crate) merge_upper_x: u32, + pub(crate) merge_upper_y: u32, + pub(crate) background: UapiProgram, + pub(crate) end_of_tile: UapiProgram, + pub(crate) partial_background: UapiProgram, + pub(crate) partial_end_of_tile: UapiProgram, + pub(crate) depth_clear: u32, + pub(crate) stencil_clear: u32, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, +} + +impl UapiRenderCommand { + fn parse(raw: &[u8]) -> Result { + let raw = padded_payload::(raw)?; + Ok(Self { + flags: get_u32(&raw, 0x00), + depth_dimensions: get_u32(&raw, 0x04), + vdm_base: get_u64(&raw, 0x08), + vertex_helper: helper(&raw, 0x10), + fragment_helper: helper(&raw, 0x20), + scissor_base: get_u64(&raw, 0x30), + depth_bias_base: get_u64(&raw, 0x38), + occlusion_query_base: get_u64(&raw, 0x40), + depth: zls(&raw, 0x48), + stencil: zls(&raw, 0x60), + zls_control: get_u64(&raw, 0x78), + multisample_control: get_u64(&raw, 0x80), + sampler_heap: get_u64(&raw, 0x88), + ppp_control: get_u32(&raw, 0x90), + width: get_u16(&raw, 0x94), + height: get_u16(&raw, 0x96), + layers: get_u16(&raw, 0x98), + sampler_count: get_u16(&raw, 0x9a), + utile_width: raw[0x9c], + utile_height: raw[0x9d], + samples: raw[0x9e], + sample_size: raw[0x9f], + merge_upper_x: get_u32(&raw, 0xa0), + merge_upper_y: get_u32(&raw, 0xa4), + background: program(&raw, 0xa8, 0xf8), + end_of_tile: program(&raw, 0xb0, 0xf0), + partial_background: program(&raw, 0xb8, 0xfc), + partial_end_of_tile: program(&raw, 0xc0, 0xf4), + depth_clear: get_u32(&raw, 0xc8), + stencil_clear: get_u32(&raw, 0xcc), + vertex_timestamps: timestamps(&raw, 0xd0), + fragment_timestamps: timestamps(&raw, 0xe0), + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiComputeCommand { + pub(crate) flags: u32, + pub(crate) sampler_count: u32, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) sampler_heap: u64, + pub(crate) helper: UapiHelperProgram, + pub(crate) timestamps: UapiTimestamps, +} + +impl UapiComputeCommand { + fn parse(raw: &[u8]) -> Result { + let raw = padded_payload::(raw)?; + Ok(Self { + flags: get_u32(&raw, 0), + sampler_count: get_u32(&raw, 4), + control_stream_base: get_u64(&raw, 8), + control_stream_end: get_u64(&raw, 0x10), + sampler_heap: get_u64(&raw, 0x18), + helper: helper(&raw, 0x20), + timestamps: timestamps(&raw, 0x30), + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ParsedHardwareCommand { + Render { + index: u32, + header: UapiCommandHeader, + payload: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + }, + Compute { + index: u32, + header: UapiCommandHeader, + payload: UapiComputeCommand, + attachments: UapiAttachmentList, + }, +} + +pub(crate) struct UapiCommandParser<'a> { + raw: &'a [u8], + offset: usize, + command_index: u32, + hardware_count: u32, + render_count: u16, + compute_count: u16, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + compute_attachments: UapiAttachmentList, +} + +impl<'a> UapiCommandParser<'a> { + pub(crate) const fn new(raw: &'a [u8]) -> Self { + Self { + raw, + offset: 0, + command_index: 0, + hardware_count: 0, + render_count: 0, + compute_count: 0, + vertex_attachments: UapiAttachmentList::EMPTY, + fragment_attachments: UapiAttachmentList::EMPTY, + compute_attachments: UapiAttachmentList::EMPTY, + } + } + + pub(crate) fn finish(&self) -> Result { + if self.offset != self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + if self.hardware_count == 0 || self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + Ok(self.hardware_count) + } + + pub(crate) fn next_hardware( + &mut self, + ) -> Result, UapiTranslateError> { + loop { + if self.offset == self.raw.len() { + return Ok(None); + } + if self.raw.len() - self.offset < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + let header = UapiCommandHeader::parse(&self.raw[self.offset..])?; + self.offset += UAPI_COMMAND_HEADER_SIZE; + let end = self + .offset + .checked_add(header.size as usize) + .ok_or(UapiTranslateError::PayloadTruncated)?; + if end > self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + let payload = &self.raw[self.offset..end]; + self.offset = end; + self.command_index += 1; + + match header.command_type { + COMMAND_RENDER | COMMAND_COMPUTE => { + if header.render_barrier != UAPI_BARRIER_NONE + && header.render_barrier > self.render_count + { + return Err(UapiTranslateError::BarrierFuture); + } + if header.compute_barrier != UAPI_BARRIER_NONE + && header.compute_barrier > self.compute_count + { + return Err(UapiTranslateError::BarrierFuture); + } + self.hardware_count += 1; + if self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + if header.command_type == COMMAND_RENDER { + self.render_count += 1; + return Ok(Some(ParsedHardwareCommand::Render { + index: self.command_index, + header, + payload: UapiRenderCommand::parse(payload)?, + vertex_attachments: self.vertex_attachments, + fragment_attachments: self.fragment_attachments, + })); + } + self.compute_count += 1; + return Ok(Some(ParsedHardwareCommand::Compute { + index: self.command_index, + header, + payload: UapiComputeCommand::parse(payload)?, + attachments: self.compute_attachments, + })); + } + SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS | SET_COMPUTE_ATTACHMENTS => { + if header.render_barrier != UAPI_BARRIER_NONE + || header.compute_barrier != UAPI_BARRIER_NONE + { + return Err(UapiTranslateError::AttachmentBarrier); + } + let attachments = UapiAttachmentList::parse(payload)?; + match header.command_type { + SET_VERTEX_ATTACHMENTS => self.vertex_attachments = attachments, + SET_FRAGMENT_ATTACHMENTS => self.fragment_attachments = attachments, + SET_COMPUTE_ATTACHMENTS => self.compute_attachments = attachments, + _ => unreachable!(), + } + } + _ => return Err(UapiTranslateError::CommandType), + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum GpuAccess { + Read, + Write, + ReadWrite, +} + +pub(crate) trait GpuAddressSpace { + fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool; +} + +fn require_range( + address_space: &A, + address: u64, + size: u64, + access: GpuAccess, +) -> Result<(), UapiTranslateError> { + let end = address + .checked_add(size) + .ok_or(UapiTranslateError::AddressRange)?; + if size == 0 || end <= address || !address_space.covers(address, size, access) { + return Err(UapiTranslateError::AddressRange); + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueueUscWindow { + pub(crate) base: u64, + pub(crate) user_start: u64, + pub(crate) user_end: u64, +} + +impl QueueUscWindow { + pub(crate) fn validate(self) -> Result<(), UapiTranslateError> { + if self.base & (USC_WINDOW_SIZE - 1) != 0 { + return Err(UapiTranslateError::QueueUscAlignment); + } + let end = self + .base + .checked_add(USC_WINDOW_SIZE) + .ok_or(UapiTranslateError::QueueUscRange)?; + if self.base < self.user_start || end > self.user_end || end <= self.base { + return Err(UapiTranslateError::QueueUscRange); + } + Ok(()) + } + + pub(crate) fn full_program( + self, + address_space: &A, + tagged_offset: u32, + ) -> Result { + self.validate()?; + if tagged_offset == 0 { + return Err(UapiTranslateError::ProgramOffset); + } + let allocation = self + .base + .checked_add((tagged_offset & !7) as u64) + .ok_or(UapiTranslateError::ProgramOffset)?; + require_range(address_space, allocation, 4, GpuAccess::Read)?; + Ok(allocation) + } +} + +fn validate_sampler( + address_space: &A, + heap: u64, + count: u32, +) -> Result<(), UapiTranslateError> { + if count > MAX_SAMPLERS { + return Err(UapiTranslateError::SamplerCount); + } + if (heap == 0) != (count == 0) { + return Err(UapiTranslateError::SamplerPair); + } + if count != 0 { + if heap & (SAMPLER_SIZE - 1) != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + heap, + count as u64 * SAMPLER_SIZE, + GpuAccess::Read, + )?; + } + Ok(()) +} + +pub(crate) fn validate_attachments( + address_space: &A, + attachments: &UapiAttachmentList, +) -> Result<(), UapiTranslateError> { + for attachment in attachments.as_slice() { + require_range( + address_space, + attachment.address, + attachment.size, + GpuAccess::Write, + )?; + } + Ok(()) +} + +pub(crate) fn validate_zls( + address_space: &A, + value: UapiZlsBuffer, + layers: u16, +) -> Result<(), UapiTranslateError> { + if value.base == 0 + && (value.compression_base != 0 || value.stride != 0 || value.compression_stride != 0) + { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_base == 0 && value.compression_stride != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if layers > 1 && value.base != 0 && value.stride == 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.stride != 0 && value.stride & 0x3fff != 1 { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_stride & 0x3fff != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.base != 0 { + let layer_stride = if value.stride == 0 { + 0 + } else { + ((value.stride as u64 >> 14) + 1) * PAGE_SIZE + }; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range(address_space, value.base, size, GpuAccess::ReadWrite)?; + } + if value.compression_base != 0 { + let layer_stride = ((value.compression_stride as u64 >> 14) + 1) * 0x80; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range( + address_space, + value.compression_base, + size, + GpuAccess::ReadWrite, + )?; + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedComputeCommand { + pub(crate) usc_exec_base: u64, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) control_stream_terminator: u64, + pub(crate) sampler_heap: u64, + pub(crate) sampler_count: u32, + pub(crate) attachments: UapiAttachmentList, + pub(crate) timestamps: UapiTimestamps, +} + +pub(crate) fn translate_compute_command( + command: UapiComputeCommand, + attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, +) -> Result { + queue.validate()?; + if command.flags != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + if command.control_stream_base & 3 != 0 + || command.control_stream_end & 3 != 0 + || command.control_stream_end <= command.control_stream_base + { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.control_stream_base, + command.control_stream_end - command.control_stream_base, + GpuAccess::Read, + )?; + validate_sampler(address_space, command.sampler_heap, command.sampler_count)?; + if command.helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + Ok(TranslatedComputeCommand { + usc_exec_base: queue.base, + control_stream_base: command.control_stream_base, + control_stream_end: command.control_stream_end, + control_stream_terminator: command.control_stream_end - 4, + sampler_heap: command.sampler_heap, + sampler_count: command.sampler_count, + attachments, + timestamps: command.timestamps, + }) +} diff --git a/drivers/gpu/drm/asahi/m3/g15_raw.rs b/drivers/gpu/drm/asahi/m3/g15_raw.rs new file mode 100644 index 0000000000000..a876cad1ec283 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/g15_raw.rs @@ -0,0 +1,1800 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU initialization / global structures + +use super::channels; +use super::types::*; +use crate::{ + default_zeroed, + gem, + mmu, + no_debug, + trivial_gpustruct, // +}; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct ChannelRing { + pub(crate) state: Option>, + pub(crate) ring: Option>, + } + + /// Host->FW ring descriptor of the 14.x firmware ABI. + /// + /// Instead of one `ChannelState` pointer, each host->FW ring (pipes, DeviceControl) is + /// described by the FW VAs of its three index words and of the ring itself. On G15 the + /// indices (and the pipe rings) live in Fender scratch SRAM. + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct ScratchRingDesc { + /// FW VA of the u32 read index (FW writes; scratch part B). +0x00 + pub(crate) read_index: U64, + /// FW VA of the u32 "CFI" index (FW writes; scratch part B). +0x08 + pub(crate) cfi_index: U64, + /// FW VA of the u32 write index (host writes; scratch part A). +0x10 + pub(crate) write_index: U64, + /// FW VA of the ring entries. +0x18 + pub(crate) ring: U64, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PipeChannelsG15V14_8_3 { + + // 14.x ABI: {TA, 3D, CL} descriptors of 0x20 bytes, 0x60 per priority + pub(crate) vtx: ScratchRingDesc, + pub(crate) frag: ScratchRingDesc, + pub(crate) comp: ScratchRingDesc, + } + default_zeroed!(PipeChannelsG15V14_8_3); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatusFlags { + pub(crate) halt_count: AtomicU64, + __pad0: Pad<0x8>, + pub(crate) halted: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) resume: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) unk_40: u32, + __pad3: Pad<0xc>, + pub(crate) unk_ctr: u32, + /// P0+0x45c4. Non-zero makes the firmware install its work-scheduler callbacks + /// Left 0, the first compute kick branches to address 0 (observed on J516S). + pub(crate) sched_callbacks: u32, + __pad4: Pad<0x8>, + pub(crate) unk_60: u32, + __pad5: Pad<0xc>, + pub(crate) unk_70: u32, + __pad6: Pad<0xc>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatus { + pub(crate) fwctl_channel: ChannelRing, + pub(crate) flags: FwStatusFlags, + } + + /// G15 "P0" globals-2/status block, 0xc3d0 bytes, pointed to by InitData+0xb0. It embeds the + /// G13 `FwStatus` at +0x4568 (FWCtl ring pointers, 8 bytes + /// of padding, then the halt/resume flags at +0x4580), which is why the + /// FWCtl ring and flags keep their `FwStatus` field names here. + #[repr(C)] + pub(crate) struct StatusBlock { + /// 0 at init; also written by the firmware register dump. +0x0 + pub(crate) unk_0: u32, + /// Noise-suppression idle power-off state. +0x4 + pub(crate) noise_suppression_idle_pwroff: u32, + /// 0 at init. +0x8 + pub(crate) unk_8: u32, + /// Not touched by the host (0x0008..0x4017). + /// TODO: firmware-private, or tables (pending stamps?) the host must fill? + pub(crate) unk_c: Array<0x400c, u8>, + /// Perf-counter sampler control, 0x4018..0x40a0. Unused by drm/asahi. + pub(crate) perfctr: Array<0x88, u8>, + /// FW-initiated recovery info (FW-written). +0x40a0/+0x40a4 + pub(crate) fw_recovery_info_a: u32, + pub(crate) fw_recovery_info_b: u32, + pub(crate) unk_40a8: Array<0x3f8, u8>, + /// Recovery-packet details read on event type 4. +0x44a0 + pub(crate) recovery_packet_info: Array<0x14, u8>, + pub(crate) unk_44b4: Array<0xc, u8>, + /// Read while draining events. +0x44c0 + pub(crate) unk_44c0: U64, + pub(crate) unk_44c8: Array<0x90, u8>, + /// Read while draining events. +0x4558/+0x4560 + pub(crate) unk_4558: U64, + pub(crate) unk_4560: U64, + /// FWCtl `{state, ring}` FW VAs. +0x4568/+0x4570 + pub(crate) fwctl_channel: ChannelRing, + /// 8 bytes of padding put the flags on a 16-byte boundary. + pub(crate) __pad_4578: Pad<0x8>, + /// halt_count/halted/resume/... at +0x4580/+0x4590/+0x45a0. + /// +0x45c4 (`flags.sched_callbacks`) must be non-zero or the work scheduler + /// is left without its callbacks. + pub(crate) flags: FwStatusFlags, + /// Not touched by the host (0x45c8..0xc3c7). + pub(crate) unk_45f0: Array<0x7dd8, u8>, + /// QoS mode, 0 at init. +0xc3c8 + pub(crate) qos_mode: u32, + /// TODO: read-modify-written at shared-data init; left 0. +0xc3cc + pub(crate) unk_c3cc: u32, + } + default_zeroed!(StatusBlock); + no_debug!(StatusBlock); + + /// G15 "P1" power-controller shared block, 0x238 bytes, InitData+0xb8. Its 0x0c..0xd4 part + /// keeps the order of the G13 13.5 `Globals` power + /// fields (power zones, fast-die and PPM controllers), so the same field names are used. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PowerCtlBlock { + /// FW-published power values. +0x0/+0x4 + pub(crate) fw_power_a: u32, + pub(crate) fw_power_b: u32, + /// Power-zone reset request. +0x8 + pub(crate) power_zone_reset: u32, + /// +0xc..+0x5f: G13 Globals `power_zone_count`..`power_zones`, same order. + pub(crate) power_zone_count: u32, + pub(crate) avg_power_filter_tc_periods: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) avg_power_target_filter_tc: u32, + pub(crate) power_zones: Array<5, PowerZoneGlobal>, + /// CLVR loop override / control effort. +0x60..+0x68 + pub(crate) clvr_loop_override_a: u32, + pub(crate) clvr_ctrl_effort: u32, + pub(crate) clvr_loop_override_b: u32, + pub(crate) unk_6c: Array<0x38, u8>, + /// Lifetime-servo temperature target. +0xa4 + pub(crate) lifetime_servo_temp_a: u32, + /// +0xa8..+0xc8: G13 Globals `unk_89bc`..`unk_89e0`, same order. + pub(crate) fast_die_temp_target: u32, + pub(crate) fast_die0_release_temp: u32, + pub(crate) unk_b0: i32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_kp: F32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) unk_c0: u32, + pub(crate) unk_c4: u32, + /// +0xc8..+0xdb: the fast-die/PPM enable flags, the PPM target (maximum power) and the + /// PPM controller gains, in G13 Globals order (`unk_89e0`, `max_power_2`, `ppm_kp`, + /// `ppm_ki_dt`). The two T6030 backends place them differently: the manager starts + /// with the enable flag at +0xc8 (the layout it was validated with), the runtime backend + /// with 0 at +0xc8 and the flag at +0xcc, as its InitData has them. See the builder. + pub(crate) ppm_words: Array<5, u32>, + pub(crate) unk_dc: Array<0xc, u8>, + /// "SE" controller override / engagement enable. +0xe8/+0xec + pub(crate) se_controller_override: u32, + pub(crate) se_engagement: u32, + pub(crate) unk_f0: Array<0x68, u8>, + /// DPE leakage update config. +0x158..+0x1ab + pub(crate) dpe_leak_cfg_a: u32, + pub(crate) unk_15c: u32, + pub(crate) unk_160: u32, + pub(crate) unk_164: u32, + pub(crate) dpe_leak_coefs: Array<0x40, u8>, + pub(crate) dpe_leak_cfg_b: u32, + pub(crate) unk_1ac: Array<0x8, u8>, + /// CLPC shared data (FW-written). +0x1b4..+0x1cc + pub(crate) clpc_shared_a: U64, + pub(crate) clpc_shared_b: U64, + pub(crate) clpc_split_ratio_a: u32, + pub(crate) clpc_split_ratio_b: u32, + pub(crate) clpc_shared_c: u32, + /// +0x1d0 (u8) / +0x1d1 (unaligned u32) + pub(crate) perf_ctrl_override_active: u8, + pub(crate) consistent_perf_state: U32, + pub(crate) unk_1d5: Array<0x3, u8>, + /// FW-written energy / filtered power. +0x1d8/+0x1e0/+0x1e4 + pub(crate) accumulated_energy: U64, + pub(crate) filtered_power: u32, + pub(crate) unk_1e4: u32, + /// SLC DSID configuration. +0x1e8..+0x22b + pub(crate) dsid_cfg_a: U64, + pub(crate) unk_1f0: Array<0x28, u8>, + pub(crate) dsid_cfg_b: U64, + pub(crate) dsid_cfg_c: U64, + pub(crate) dsid_cfg_d: u32, + pub(crate) unk_22c: u32, + /// +0x230 + pub(crate) submit_in_progress_gate: u32, + pub(crate) unk_234: u32, + } + default_zeroed!(PowerCtlBlock); + + /// G15 "P2" debug/trace block, 0x20 bytes, InitData+0xa8. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct DebugBlock { + /// Moved here from G13 `Globals.ktrace_enable`. +0x0 + pub(crate) ktrace_enable: u32, + /// TODO: written at init with an unknown value; left 0. +0x4 + pub(crate) unk_4: u32, + /// System-sleep notification in progress. +0x8 + pub(crate) system_sleep_in_progress: u32, + pub(crate) unk_c: u32, + /// Firmware-written. Read-only for the host. + pub(crate) fw_word_10: AtomicU32, + /// Firmware init progress: the firmware writes 0 once it consumes the INIT handshake and 1 + /// once its post-INIT initialisation is done, each followed by a barrier. The G15 driver + /// writes a sentinel here before MSG_INIT so the three states (not consumed / stuck / done) + /// are distinguishable. + pub(crate) init_state: AtomicU32, + pub(crate) unk_18: u32, + pub(crate) unk_1c: u32, + } + default_zeroed!(DebugBlock); + + /// One Globals register-override record, 0x18 bytes. + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct RegisterOverride { + pub(crate) addr: U64, + pub(crate) value: U64, + pub(crate) id: u32, + pub(crate) __pad: u32, + } + + /// One InitSeq register-initialisation record, 0x18 bytes. + /// + /// The InitSeq buffer (InitData+0x08) is a list of these. A zeroed buffer is a valid empty + /// list, which is what drm/asahi passes; this type only documents the format. + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + #[allow(dead_code)] + pub(crate) struct InitSeqRecord { + pub(crate) value: U64, + pub(crate) reg_id: u32, + pub(crate) extra: u32, + /// 1 = 32-bit, 2 = 64-bit, 3 = 64-bit physical address. + pub(crate) kind: u32, + pub(crate) __pad: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared1 { + pub(crate) table: Array<16, i32>, + pub(crate) unk_44: Array<0x60, u8>, + pub(crate) unk_a4: u32, + pub(crate) unk_a8: u32, + } + default_zeroed!(HwDataShared1); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2Curve { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) t1: Array<16, u16>, + pub(crate) t2: Array<16, i16>, + pub(crate) t3: Array<8, Array<16, i32>>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2G14 { + pub(crate) unk_0: Array<5, u32>, + pub(crate) unk_14: u32, + pub(crate) unk_18: Array<8, u32>, + pub(crate) curve1: HwDataShared2Curve, + pub(crate) curve2: HwDataShared2Curve, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared2 { + pub(crate) table: Array<10, i32>, + pub(crate) unk_28: Array<0x10, u8>, + pub(crate) g14: HwDataShared2G14, + pub(crate) unk_500: u32, + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + } + default_zeroed!(HwDataShared2); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared3 { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) unk_8: u32, + pub(crate) table: Array<16, u32>, + pub(crate) unk_4c: u32, + } + default_zeroed!(HwDataShared3); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataA130Extra { + pub(crate) unk_0: Array<0x38, u8>, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) gpu_se_inactive_threshold: u32, + pub(crate) unk_44: u32, + pub(crate) gpu_se_engagement_criteria: i32, + pub(crate) gpu_se_reset_criteria: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) gpu_se_filter_a_neg: F32, + pub(crate) gpu_se_filter_1_a_neg: F32, + pub(crate) gpu_se_filter_a: F32, + pub(crate) gpu_se_filter_1_a: F32, + pub(crate) gpu_se_ki_dt: F32, + pub(crate) gpu_se_ki_1_dt: F32, + pub(crate) unk_78: F32, + pub(crate) unk_7c: F32, + pub(crate) gpu_se_kp: F32, + pub(crate) gpu_se_kp_1: F32, + pub(crate) unk_88: u32, + pub(crate) unk_8c: u32, + pub(crate) max_pstate_scaled_1: u32, + pub(crate) unk_94: u32, + pub(crate) unk_98: u32, + pub(crate) unk_9c: F32, + pub(crate) unk_a0: u32, + pub(crate) unk_a4: u32, + pub(crate) gpu_se_filter_time_constant_ms: u32, + pub(crate) gpu_se_filter_time_constant_1_ms: u32, + pub(crate) gpu_se_filter_time_constant_clks: U64, + pub(crate) gpu_se_filter_time_constant_1_clks: U64, + pub(crate) unk_c0: u32, + pub(crate) unk_c4: F32, + pub(crate) unk_c8: Array<0x4c, u8>, + pub(crate) unk_114: F32, + pub(crate) unk_118: u32, + pub(crate) unk_11c: u32, + pub(crate) unk_120: u32, + pub(crate) unk_124: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_12c: Array<0x8c, u8>, + } + default_zeroed!(HwDataA130Extra); + + /// G15 (14.x) shader-engine controller block of HwDataA, at +0x10b4. + /// + /// It holds the fields of [`HwDataA130Extra`] in a different order: the filter, gain and + /// time-constant words move 0x14 bytes down, and the threshold/criteria header moves behind + /// them, to +0xd8. Offsets in the field names are G15 block offsets. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataA140Extra { + pub(crate) unk_0: Array<0x40, u8>, + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + pub(crate) unk_48: u32, + pub(crate) gpu_se_filter_a_neg: F32, + pub(crate) gpu_se_filter_1_a_neg: F32, + pub(crate) gpu_se_filter_a: F32, + pub(crate) gpu_se_filter_1_a: F32, + pub(crate) gpu_se_ki_dt: F32, + pub(crate) gpu_se_ki_1_dt: F32, + pub(crate) unk_64: F32, + pub(crate) unk_68: F32, + pub(crate) gpu_se_kp: F32, + pub(crate) gpu_se_kp_1: F32, + pub(crate) unk_74: u32, + pub(crate) unk_78: u32, + pub(crate) max_pstate_scaled_1: u32, + pub(crate) min_pstate_scaled: u32, + pub(crate) unk_84: u32, + pub(crate) unk_88: F32, + /// Shader-engine target (ADT gpu-se-tgt). + pub(crate) se_target: u32, + pub(crate) unk_90: u32, + pub(crate) gpu_se_filter_time_constant_ms: u32, + pub(crate) gpu_se_filter_time_constant_1_ms: u32, + pub(crate) gpu_se_filter_time_constant_clks: U64, + pub(crate) gpu_se_filter_time_constant_1_clks: U64, + pub(crate) unk_ac: u32, + pub(crate) unk_b0: F32, + pub(crate) unk_b4: Array<0x24, u8>, + pub(crate) unk_d8: u32, + pub(crate) unk_dc: u32, + pub(crate) gpu_se_inactive_threshold: u32, + pub(crate) gpu_se_engagement_criteria: i32, + pub(crate) unk_e8: u32, + pub(crate) unk_ec: u32, + pub(crate) gpu_se_reset_criteria: u32, + pub(crate) unk_f4: Array<0x28, u8>, + pub(crate) unk_11c: F32, + pub(crate) unk_120: Array<0xc, u8>, + pub(crate) unk_12c: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_134: Array<0x84, u8>, + } + default_zeroed!(HwDataA140Extra); + + #[repr(C)] + pub(crate) struct T81xxData { + pub(crate) unk_d8c: u32, + pub(crate) unk_d90: u32, + pub(crate) unk_d94: u32, + pub(crate) unk_d98: u32, + pub(crate) unk_d9c: F32, + pub(crate) unk_da0: u32, + pub(crate) unk_da4: F32, + pub(crate) unk_da8: u32, + pub(crate) unk_dac: F32, + pub(crate) unk_db0: u32, + pub(crate) unk_db4: u32, + pub(crate) unk_db8: F32, + pub(crate) unk_dbc: F32, + pub(crate) unk_dc0: u32, + pub(crate) unk_dc4: u32, + pub(crate) unk_dc8: u32, + pub(crate) max_pstate_scaled: u32, + } + default_zeroed!(T81xxData); + + #[derive(Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct PowerZoneG15V14_8_3 { + pub(crate) val: F32, + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc_x4: u32, + pub(crate) filter_tc_xperiod: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: u32, + pub(crate) filter_a_neg: F32, + pub(crate) filter_a: F32, + pub(crate) pad: u32, + } + + // G15 HwDataA uses the G13 single-die layout with 8x8 coefficient tables at 0x970-0xb6f + //, so the G14X 16-core layout is G14X-only. + const MAX_CORES_PER_CLUSTERG15V14_8_3: usize = { + { + 8 + } + }; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct AuxLeakCoef { + pub(crate) afr_1: Array<2, F32>, + pub(crate) cs_1: Array<2, F32>, + pub(crate) afr_2: Array<2, F32>, + pub(crate) cs_2: Array<2, F32>, + } + + #[repr(C)] + pub(crate) struct HwDataAG15V14_8_3 { + pub(crate) unk_0: u32, + pub(crate) clocks_per_period: u32, + + pub(crate) clocks_per_period_2: u32, + + pub(crate) unk_8: u32, + pub(crate) pwr_status: AtomicU32, + pub(crate) unk_10: F32, + pub(crate) unk_14: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, + pub(crate) actual_pstate: u32, + pub(crate) tgt_pstate: u32, + pub(crate) unk_30: u32, + pub(crate) cur_pstate: u32, + pub(crate) unk_38: u32, + + pub(crate) unk_3c_0: u32, + + pub(crate) base_pstate_scaled: u32, + pub(crate) unk_40: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) unk_48: u32, + pub(crate) min_pstate_scaled: u32, + pub(crate) freq_mhz: F32, + pub(crate) unk_54: Array<0x20, u8>, + + pub(crate) unk_74_0: u32, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_b4: Array<0x100, u8>, + pub(crate) unk_1b4: u32, + pub(crate) temp_c: u32, + pub(crate) avg_power_mw: u32, + pub(crate) update_ts: U64, + pub(crate) unk_1c8: u32, + pub(crate) unk_1cc: Array<0x478, u8>, + pub(crate) pad_644: Pad<0x8>, + pub(crate) unk_64c: u32, + pub(crate) unk_650: u32, + pub(crate) pad_654: u32, + pub(crate) pwr_filter_a_neg: F32, + pub(crate) pad_65c: u32, + pub(crate) pwr_filter_a: F32, + pub(crate) pad_664: u32, + pub(crate) pwr_integral_gain: F32, + pub(crate) pad_66c: u32, + pub(crate) pwr_integral_min_clamp: F32, + pub(crate) max_power_1: F32, + pub(crate) pwr_proportional_gain: F32, + pub(crate) pad_67c: u32, + pub(crate) pwr_pstate_related_k: F32, + pub(crate) pwr_pstate_max_dc_offset: i32, + pub(crate) unk_688: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) pad_690: u32, + pub(crate) unk_694: u32, + pub(crate) max_power_2: u32, + pub(crate) pad_69c: Pad<0x18>, + pub(crate) unk_6b4: u32, + + pub(crate) unk_6b8_0: Array<0x10, u8>, + + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_6bc: u32, + pub(crate) pad_6c0: Pad<0x14>, + pub(crate) ppm_filter_tc_periods_x4: u32, + pub(crate) unk_6d8: u32, + pub(crate) pad_6dc: u32, + pub(crate) ppm_filter_a_neg: F32, + pub(crate) pad_6e4: u32, + pub(crate) ppm_filter_a: F32, + pub(crate) pad_6ec: u32, + pub(crate) ppm_ki_dt: F32, + pub(crate) pad_6f4: u32, + pub(crate) pwr_integral_min_clamp_2: u32, + pub(crate) unk_6fc: F32, + pub(crate) ppm_kp: F32, + pub(crate) pad_704: u32, + pub(crate) unk_708: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_4: u32, + pub(crate) unk_714: u32, + pub(crate) pad_718: u32, + pub(crate) unk_71c: F32, + pub(crate) max_power_3: u32, + pub(crate) cur_power_mw_2: u32, + pub(crate) ppm_filter_tc_ms: u32, + pub(crate) unk_72c: u32, + + pub(crate) ppm_filter_tc_clks: u32, + + pub(crate) unk_730_4: u32, + + pub(crate) unk_730_8: u32, + + pub(crate) unk_730_c: u32, + + pub(crate) unk_730: F32, + pub(crate) unk_734: u32, + pub(crate) unk_738: u32, + pub(crate) unk_73c: u32, + pub(crate) unk_740: u32, + pub(crate) unk_744: u32, + pub(crate) unk_748: Array<0x4, F32>, + pub(crate) unk_758: u32, + pub(crate) perf_tgt_utilization: u32, + pub(crate) pad_760: u32, + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_boost_ce_step: u32, + pub(crate) perf_reset_iters: u32, + pub(crate) pad_770: u32, + pub(crate) unk_774: u32, + pub(crate) unk_778: u32, + pub(crate) perf_filter_drop_threshold: u32, + pub(crate) perf_filter_a_neg: F32, + pub(crate) perf_filter_a2_neg: F32, + pub(crate) perf_filter_a: F32, + pub(crate) perf_filter_a2: F32, + pub(crate) perf_ki: F32, + pub(crate) perf_ki2: F32, + pub(crate) perf_integral_min_clamp: F32, + pub(crate) unk_79c: F32, + pub(crate) perf_kp: F32, + pub(crate) perf_kp2: F32, + pub(crate) boost_state_unk_k: F32, + pub(crate) base_pstate_scaled_2: u32, + pub(crate) max_pstate_scaled_5: u32, + pub(crate) base_pstate_scaled_3: u32, + pub(crate) pad_7b8: u32, + pub(crate) perf_cur_utilization: F32, + pub(crate) perf_tgt_utilization_2: u32, + pub(crate) pad_7c4: Pad<0x18>, + pub(crate) unk_7dc: u32, + + pub(crate) unk_7e0_0: Array<0x10, u8>, + + pub(crate) base_pstate_scaled_4: u32, + pub(crate) pad_7e4: u32, + pub(crate) unk_7e8: Array<0x14, u8>, + pub(crate) unk_7fc: F32, + pub(crate) pwr_min_duty_cycle_2: F32, + pub(crate) max_pstate_scaled_6: F32, + pub(crate) max_freq_mhz: u32, + pub(crate) pad_80c: u32, + pub(crate) unk_810: u32, + pub(crate) pad_814: u32, + pub(crate) pwr_min_duty_cycle_3: u32, + pub(crate) unk_81c: u32, + pub(crate) pad_820: u32, + pub(crate) min_pstate_scaled_4: F32, + pub(crate) max_pstate_scaled_7: u32, + pub(crate) unk_82c: u32, + pub(crate) unk_alpha_neg: F32, + pub(crate) unk_alpha: F32, + pub(crate) unk_838: u32, + pub(crate) unk_83c: u32, + pub(crate) pad_840: Pad<0x2c>, + pub(crate) unk_86c: u32, + pub(crate) fast_die0_sensor_mask: U64, + pub(crate) fast_die0_release_temp_cc: u32, + pub(crate) unk_87c: i32, + pub(crate) unk_880: u32, + pub(crate) unk_884: u32, + pub(crate) pad_888: u32, + pub(crate) unk_88c: u32, + pub(crate) pad_890: u32, + pub(crate) unk_894: F32, + pub(crate) pad_898: u32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) pad_8a0: u32, + pub(crate) unk_8a4: u32, + pub(crate) unk_8a8: F32, + pub(crate) fast_die0_kp: F32, + pub(crate) pad_8b0: u32, + pub(crate) unk_8b4: u32, + pub(crate) pwr_min_duty_cycle_4: u32, + pub(crate) max_pstate_scaled_8: u32, + pub(crate) max_pstate_scaled_9: u32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) unk_8c8: u32, + pub(crate) unk_8cc: u32, + pub(crate) pad_8d0: Pad<0x14>, + + pub(crate) unk_8e4_0: Array<0x10, u8>, + + pub(crate) unk_8e4: u32, + pub(crate) unk_8e8: u32, + pub(crate) max_pstate_scaled_10: u32, + pub(crate) unk_8f0: u32, + pub(crate) unk_8f4: u32, + pub(crate) pad_8f8: u32, + pub(crate) pad_8fc: u32, + pub(crate) unk_900: Array<0x24, u8>, + + pub(crate) unk_coef_a1: Array<8, Array>, + pub(crate) unk_coef_a2: Array<8, Array>, + + pub(crate) pad_b24: Pad<0x70>, + pub(crate) max_pstate_scaled_11: u32, + pub(crate) freq_with_off: u32, + pub(crate) unk_b9c: u32, + pub(crate) unk_ba0: U64, + pub(crate) unk_ba8: U64, + pub(crate) unk_bb0: u32, + pub(crate) unk_bb4: u32, + + pub(crate) pad_bb8_0: Pad<0x200>, + pub(crate) pad_bb8_200: Pad<0x8>, + + pub(crate) pad_bb8: Pad<0x74>, + pub(crate) unk_c2c: u32, + pub(crate) power_zone_count: u32, + pub(crate) max_power_4: u32, + pub(crate) max_power_5: u32, + pub(crate) max_power_6: u32, + pub(crate) unk_c40: u32, + pub(crate) unk_c44: F32, + pub(crate) avg_power_target_filter_a_neg: F32, + pub(crate) avg_power_target_filter_a: F32, + pub(crate) avg_power_target_filter_tc_x4: u32, + pub(crate) avg_power_target_filter_tc_xperiod: u32, + + pub(crate) avg_power_target_filter_tc_clks: u32, + + pub(crate) unk_c58_4: u32, + + pub(crate) power_zones: Array<5, PowerZoneG15V14_8_3>, + pub(crate) avg_power_filter_tc_periods_x4: u32, + pub(crate) unk_cfc: u32, + pub(crate) unk_d00: u32, + pub(crate) avg_power_filter_a_neg: F32, + pub(crate) unk_d08: u32, + pub(crate) avg_power_filter_a: F32, + pub(crate) unk_d10: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) unk_d18: u32, + pub(crate) unk_d1c: u32, + pub(crate) unk_d20: F32, + pub(crate) avg_power_kp: F32, + pub(crate) unk_d28: u32, + pub(crate) unk_d2c: u32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_12: u32, + pub(crate) max_pstate_scaled_13: u32, + pub(crate) unk_d3c: u32, + pub(crate) max_power_7: F32, + pub(crate) max_power_8: u32, + pub(crate) unk_d48: u32, + pub(crate) avg_power_filter_tc_ms: u32, + pub(crate) unk_d50: u32, + + pub(crate) avg_power_filter_tc_clks: u32, + + pub(crate) unk_d54_4: Array<0xc, u8>, + + pub(crate) unk_d54: Array<0x10, u8>, + pub(crate) max_pstate_scaled_14: u32, + pub(crate) unk_d68: Array<0x24, u8>, + + pub(crate) t81xx_data: T81xxData, + + pub(crate) unk_dd0: Array<0x40, u8>, + + pub(crate) unk_e10_pad: Array<0x10, u8>, + pub(crate) unk_e10_0: HwDataA140Extra, + + pub(crate) unk_e10: Array<0xc, u8>, + + // G15 offsets match G13 13.5 up to ~0x1224 and are +0x10 from 0x1288 + // (`fast_die0_sensor_mask_2`, G13 0x1278). TODO: exact insertion point between + // 0x1224 and 0x1288 is not known; placed as late as possible. + pub(crate) unk_1278_g15: Array<0x10, u8>, + + pub(crate) fast_die0_sensor_mask_2: U64, + + pub(crate) unk_e24: u32, + pub(crate) unk_e28: u32, + pub(crate) unk_e2c: Pad<0x1c>, + pub(crate) unk_coef_b1: Array<8, Array>, + pub(crate) unk_coef_b2: Array<8, Array>, + + pub(crate) pad_1048: Pad<0x5e4>, + + pub(crate) fast_die0_sensor_mask_alt: U64, + + pub(crate) unk_163c: u32, + + pub(crate) unk_1640: Array<0x2000, u8>, + + pub(crate) unk_3640: u32, + pub(crate) unk_3644: u32, + pub(crate) hws1: HwDataShared1, + + pub(crate) unk_hws2: Array<16, u16>, + + pub(crate) hws2: HwDataShared2, + pub(crate) unk_3c00: u32, + pub(crate) unk_3c04: u32, + pub(crate) hws3: HwDataShared3, + pub(crate) unk_3c58: Array<0x3c, u8>, + pub(crate) unk_3c94: u32, + pub(crate) unk_3c98: U64, + pub(crate) unk_3ca0: U64, + pub(crate) unk_3ca8: U64, + pub(crate) unk_3cb0: U64, + pub(crate) ts_last_idle: U64, + pub(crate) ts_last_poweron: U64, + pub(crate) ts_last_poweroff: U64, + pub(crate) unk_3cd0: U64, + pub(crate) unk_3cd8: U64, + + pub(crate) unk_3ce0_0: u32, + + pub(crate) unk_3ce0: u32, + pub(crate) unk_3ce4: u32, + pub(crate) unk_3ce8: u32, + pub(crate) unk_3cec: u32, + pub(crate) unk_3cf0: u32, + pub(crate) core_leak_coef: Array<8, F32>, + pub(crate) sram_leak_coef: Array<8, F32>, + + pub(crate) aux_leak_coef: AuxLeakCoef, + pub(crate) unk_3d34_0: Array<0x18, u8>, + + pub(crate) unk_3d34: Array<0x38, u8>, + + // G15 tail, 0x422c..0x4360. + pub(crate) unk_422c: Array<0x20, u8>, + + /// System counter value at init. +0x424c + /// The runtime backend fills it; the manager leaves it 0, as validated. + pub(crate) init_timestamp: U64, + pub(crate) unk_4254: Array<0x40, u8>, + + /// +0x4294/+0x4298 + pub(crate) unk_4294: u32, + pub(crate) unk_4298: u32, + pub(crate) unk_429c: Array<0x8, u8>, + + /// Three 8-entry per-cluster tables. +0x42a4 + /// TODO: source values unknown; left 0. + pub(crate) cluster_tables: Array<3, Array<8, u32>>, + + /// Eight words, source unknown. +0x4304..+0x4320 + pub(crate) unk_4304: Array<8, u32>, + pub(crate) unk_4324: Array<0x1c, u8>, + + /// Read-modify-written at init. +0x4340 + pub(crate) unk_4340: U64, + pub(crate) unk_4348: Array<0x10, u8>, + + /// GPU keep-alive mode (read by the host). +0x4358 + pub(crate) gpu_keepalive_mode: u32, + pub(crate) unk_435c: u32, + } + default_zeroed!(HwDataAG15V14_8_3); + no_debug!(HwDataAG15V14_8_3); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C)] + pub(crate) struct IOMapping { + pub(crate) phys_addr: U64, + pub(crate) virt_addr: U64, + pub(crate) total_size: u32, + pub(crate) element_size: u32, + pub(crate) readwrite: U64, + } + + const IO_MAPPING_COUNTG15V14_8_3: usize = { + // 31 IOMapping slots (0x3e0 / 0x20) + { + 0x1f + } + }; + + /// A G15 performance-state table: the highest entry, then per entry the frequency (MHz) + /// and the core and SRAM voltages (mV), eight voltage columns each. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBPStateTable { + pub(crate) max_pstate: u32, + pub(crate) frequencies: Array<0x10, u32>, + pub(crate) voltages: Array<0x10, [u32; 0x8]>, + pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBAuxPStates { + pub(crate) cs_max_pstate: u32, + pub(crate) cs_frequencies: Array<0x10, u32>, + pub(crate) cs_voltages: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_voltages_sram: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_unkpad: u32, + pub(crate) afr_max_pstate: u32, + pub(crate) afr_frequencies: Array<0x8, u32>, + pub(crate) afr_voltages: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_voltages_sram: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_unkpad: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBG15V14_8_3 { + + pub(crate) unk_8: U64, + + pub(crate) unk_18: U64, + pub(crate) unk_20: U64, + pub(crate) unk_28: U64, + pub(crate) unk_30: U64, + pub(crate) timestamp_area_base: U64, + + /// G15: a pointer. +0x30 + /// TODO: what it points to; left 0. + pub(crate) unk_30_ptr: U64, + + // G15: two tables of 32 CSC matrices (3x4 i16) at 0x38..0x637 + pub(crate) yuv_matrices: Array<0x40, Array<3, Array<4, i16>>>, + + pub(crate) pad_1c8: Pad<0x8>, + pub(crate) io_mappings: Array, + + pub(crate) sgx_sram_ptr: U64, + + pub(crate) chip_id: u32, + pub(crate) unk_454: u32, + pub(crate) unk_458: u32, + pub(crate) unk_45c: u32, + pub(crate) unk_460: u32, + pub(crate) unk_464: u32, + pub(crate) unk_468: u32, + pub(crate) unk_46c: u32, + pub(crate) unk_470: u32, + pub(crate) unk_474: u32, + pub(crate) unk_478: u32, + pub(crate) unk_47c: u32, + pub(crate) unk_480: u32, + pub(crate) unk_484: u32, + pub(crate) unk_488: u32, + pub(crate) unk_48c: u32, + pub(crate) base_clock_khz: u32, + pub(crate) power_sample_period: u32, + + // G15 0xa70..0xb3f. The G13 13.5 0x9b0..0xa73 group, + // with 12 bytes inserted at an unknown position; the words are named by their G15 offset. + // TODO: map these to the G13 names (`num_frags`, `unk_524` secure cache flush, ...) + // once their values are known. + pub(crate) unk_a70: u32, + pub(crate) unk_a74: u32, + pub(crate) unk_a78: u32, + pub(crate) unk_a7c: U64, + pub(crate) unk_a84: u32, + pub(crate) unk_a88: u32, + pub(crate) unk_a8c: u32, + pub(crate) unk_a90: u32, + pub(crate) unk_a94: u32, + pub(crate) unk_a98: u32, + pub(crate) unk_a9c: u32, + pub(crate) unk_aa0: u32, + pub(crate) unk_aa4: u32, + pub(crate) unk_aa8: u32, + pub(crate) unk_aac: u32, + pub(crate) unk_ab0: u32, + pub(crate) unk_ab4: u32, + pub(crate) unk_ab8: u32, + pub(crate) unk_abc: U64, + pub(crate) unk_ac4: u32, + pub(crate) unk_ac8: Array<0x4, u32>, + pub(crate) unk_ad8: u32, + pub(crate) unk_adc: u32, + pub(crate) unk_ae0: u32, + pub(crate) unk_ae4: u32, + pub(crate) unk_ae8: U64, + pub(crate) unk_af0: u32, + pub(crate) unk_af4: U64, + pub(crate) unk_afc: u32, + pub(crate) unk_b00: u32, + pub(crate) unk_b04: u32, + pub(crate) unk_b08: Array<0x4, u32>, + pub(crate) unk_b18: u32, + pub(crate) unk_b1c: u32, + pub(crate) unk_b20: u32, + pub(crate) unk_b24: U64, + pub(crate) unk_b2c: Array<0x14, u8>, + + pub(crate) unk_554: u32, + pub(crate) uat_ttb_base: U64, + pub(crate) gpu_core_id: u32, + pub(crate) gpu_rev_id: u32, + pub(crate) num_cores: u32, + pub(crate) max_pstate: u32, + + pub(crate) frequencies: Array<0x10, u32>, + pub(crate) voltages: Array<0x10, [u32; 0x8]>, + pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + + + /// G15: second 16-entry frequency table in MHz (meaning inferred). +0xf9c + pub(crate) frequencies_2: Array<0x10, u32>, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_9f4: Array<0x10, u32>, + pub(crate) rel_max_powers: Array<0x10, u32>, + pub(crate) rel_boost_freqs: Array<0x10, u32>, + + pub(crate) unk_arr_0: Array<32, u32>, + + pub(crate) aux_ps: HwDataBAuxPStates, + + // G15 0x134c..0x17eb: a third p-state table at 0x134c, then the + // words before `timer_offset`. G15 is +4 relative to G13 here (+0xd0 at `timer_offset`). + + /// Third p-state table: the secondary (lowest frequency per voltage) table, with one + /// voltage column. +0x134c + pub(crate) aux_ps_3: HwDataBPStateTable, + pub(crate) aux_ps_3_pad: Array<0xc, u8>, + pub(crate) unk_179c: U64, + pub(crate) unk_17a4: Array<0x10, u8>, + pub(crate) unk_17b4: u32, + pub(crate) unk_17b8: u32, + pub(crate) unk_17bc: u32, + + /// Unit enable bit masks. +0x17c0/+0x17c8 + pub(crate) unit_mask_a: U64, + pub(crate) unit_mask_b: u32, + pub(crate) unk_17cc: u32, + pub(crate) unk_17d0: u32, + pub(crate) unk_17d4: Array<0x10, u8>, + pub(crate) unk_17e4: u32, + pub(crate) unk_17e8: u32, + + pub(crate) timer_offset: U64, + + // G15 0x17f4..0x1867. + pub(crate) unk_17f4: U64, + pub(crate) unk_17fc: u32, + pub(crate) unk_1800: u32, + pub(crate) unk_1804: u32, + pub(crate) unk_1808: u32, + pub(crate) unk_180c: u32, + + /// +0x1810 and +0x1814 are both 1 (written as one 64-bit store); the firmware copies them + /// to separate globals. + pub(crate) unk_1810: u32, + pub(crate) unk_1814: u32, + pub(crate) unk_1818: u32, + pub(crate) unk_181c: u32, + + /// 48 bytes of 0xff ("all units enabled" masks?). +0x1820 + pub(crate) all_ones_masks: Array<0x30, u8>, + + /// FSTP override enable. +0x1850 + pub(crate) fstp_override: u32, + pub(crate) unk_1854: U64, + pub(crate) unk_185c: u32, + pub(crate) unk_1860: u32, + pub(crate) unk_1864: u32, + } + default_zeroed!(HwDataBG15V14_8_3); + + #[derive(Debug)] + #[repr(C, packed)] + pub(crate) struct GpuStatsVtx { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsVtx); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsFragG15V14_8_3 { + // G15: the two -1 fields are at E2+0xc18/+0xc30 + pub(crate) unk1_0: Array<0xb10, u8>, + pub(crate) unk1: Array<0x100, u8>, + pub(crate) cur_stamp_id: i32, + pub(crate) unk2: Array<0x14, u8>, + pub(crate) unk_id: i32, + // G15 E2 is 0x1248 bytes in total + pub(crate) unk3: Array<0x614, u8>, + } + + impl Default for GpuStatsFragG15V14_8_3 { + fn default() -> Self { + Self { + unk1_0: Default::default(), + unk1: Default::default(), + cur_stamp_id: -1, + unk2: Default::default(), + unk_id: -1, + unk3: Default::default(), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsVtx { + pub(crate) total_cmds: u32, + pub(crate) stats: GpuStatsVtx, + } + default_zeroed!(GpuGlobalStatsVtx); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsFragG15V14_8_3 { + pub(crate) total_cmds: u32, + pub(crate) unk_4: u32, + pub(crate) stats: GpuStatsFragG15V14_8_3, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsComp { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsComp); + + // Not part of the 0x490-byte 14.x RuntimePointers, so the G15V14_8_3 + // instantiation is unused. + #[derive(Debug)] + #[repr(C)] + #[allow(dead_code)] + pub(crate) struct RuntimeScratchG15V14_8_3 { + pub(crate) unk_280: Array<0x6800, u8>, + pub(crate) unk_6a80: u32, + pub(crate) gpu_idle: u32, + pub(crate) unkpad_6a88: Pad<0x14>, + pub(crate) unk_6a9c: u32, + pub(crate) unk_ctr0: u32, + pub(crate) unk_ctr1: u32, + pub(crate) unk_6aa8: u32, + pub(crate) unk_6aac: u32, + pub(crate) unk_ctr2: u32, + pub(crate) unk_6ab4: u32, + pub(crate) unk_6ab8: u32, + pub(crate) unk_6abc: u32, + pub(crate) unk_6ac0: u32, + pub(crate) unk_6ac4: u32, + pub(crate) unk_ctr3: u32, + pub(crate) unk_6acc: u32, + pub(crate) unk_6ad0: u32, + pub(crate) unk_6ad4: u32, + pub(crate) unk_6ad8: u32, + pub(crate) unk_6adc: u32, + pub(crate) unk_6ae0: u32, + pub(crate) unk_6ae4: u32, + pub(crate) unk_6ae8: u32, + pub(crate) unk_6aec: u32, + pub(crate) unk_6af0: u32, + pub(crate) unk_ctr4: u32, + pub(crate) unk_ctr5: u32, + pub(crate) unk_6afc: u32, + pub(crate) pad_6b00: Pad<0x38>, + + pub(crate) pad_6b00_extra: Array<0x4800, u8>, + + pub(crate) unk_6b38: u32, + pub(crate) pad_6b3c: Pad<0x84>, + } + default_zeroed!(RuntimeScratchG15V14_8_3); + + #[repr(C)] + pub(crate) struct RuntimePointersG15V14_8_3<'a> { + // G15 / 14.8.3: 0x490 bytes, packed (HwDataA pointer at +0x441). GpuPointer/U64 are + // packed(1), so repr(C) gives the firmware offsets. + + /// +0x000 + pub(crate) hwdata_b: GpuPointer<'a, super::HwDataBG15V14_8_3>, + + /// Hardware-erratum (BRN) table. +0x008 + pub(crate) brn_table: GpuPointer<'a, &'a [u8]>, + + /// E7 (0x88 bytes, zeroed). +0x010 + pub(crate) unkptr_e7: GpuPointer<'a, &'a [u8]>, + + pub(crate) pipes: Array<4, PipeChannelsG15V14_8_3>, + + /// +0x198 + pub(crate) device_control: ScratchRingDesc, + pub(crate) event: ChannelRing, + pub(crate) fw_log: ChannelRing, + pub(crate) ktrace: ChannelRing, + pub(crate) stats: ChannelRing, + + + /// FWLog payload buffer (0x600 x 0xd8). +0x1f8 + pub(crate) fwlog_buf: Option>, + pub(crate) __pad_200: Pad<0x30>, + + /// 0 at init. +0x230 + pub(crate) unk_230: u32, + pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx>, + pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFragG15V14_8_3>, + pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>, + + + /// E5 (0x60 bytes, zeroed; FW status word at +0). +0x24c + pub(crate) unkptr_e5: GpuPointer<'a, &'a [u8]>, + pub(crate) __pad_254: Pad<0x54>, + + /// Optional pointer, left 0. +0x2a8 + pub(crate) unkptr_2a8: U64, + + /// PB descriptor table GPU/FW VA. +0x2b0/+0x2b8 + pub(crate) buffer_mgr_ctl_gpu_addr: U64, + pub(crate) buffer_mgr_ctl_fw_addr: U64, + + /// UMA page-pool descriptor table GPU/FW VA. +0x2c0/+0x2c8 + pub(crate) uma_pool_desc_gpu_addr: U64, + pub(crate) uma_pool_desc_fw_addr: U64, + + /// Configuration words, left 0. +0x2d0/+0x2d4 + pub(crate) unk_2d0: u32, + pub(crate) unk_2d4: u32, + pub(crate) __pad_2d8: Pad<0xd8>, + + /// 0xff at init (context-switch histogram control). +0x3b0 + pub(crate) cswitch_hist_ctl: u8, + pub(crate) unk_3b1: Array<0x90, u8>, + + // G15: +0x441 (unaligned) + pub(crate) hwdata_a: GpuPointer<'a, super::HwDataAG15V14_8_3>, + + + /// 16 configuration bytes, left 0. +0x449 + pub(crate) unk_449: Array<0x10, u8>, + pub(crate) __pad_459: Pad<0x14>, + + /// Zero at init. +0x46d + pub(crate) unk_46d: Array<0x20, u8>, + pub(crate) __pad_48d: Pad<0x3>, + } + no_debug!(RuntimePointersG15V14_8_3<'_>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PendingStamp { + pub(crate) info: AtomicU32, + pub(crate) wait_value: AtomicU32, + } + default_zeroed!(PendingStamp); + + #[derive(Debug, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct FaultInfo { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) queue_uuid: u32, + pub(crate) unk_c: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: u32, + } + default_zeroed!(FaultInfo); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct PowerZoneGlobal { + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc: u32, + } + default_zeroed!(PowerZoneGlobal); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GlobalsG15V14_8_3 { + // G15 / 14.8.3 layout: 0xe00 bytes, cached, packed. + // It re-packs the G13 field groups; the large G13 arrays are gone. + + /// Timer context-switch mode, 0 or 7. +0x0 + pub(crate) timer_cswitch: u32, + pub(crate) unk_4: u32, + + /// +0x8 + pub(crate) dm_pause_mode: u32, + pub(crate) unk_c: u32, + + /// +0x10 + pub(crate) dm_pause_timer: u32, + pub(crate) unk_14: u32, + + /// +0x18 + pub(crate) disable_cswitch: u32, + + /// Configuration word. +0x1c + pub(crate) unk_1c: u32, + + /// +0x20/+0x24 + pub(crate) relaxed_cl_kill_timeout: u32, + pub(crate) frg_task_timeout: u32, + + /// G13 `unk_24`. +0x28 + pub(crate) unk_28: u32, + + /// Debug flags bit 0 (G13 `debug`). +0x2c + pub(crate) debug: u32, + + /// +0x30 + pub(crate) smart_idle_off_enable: u32, + pub(crate) unk_34: u32, + + /// Constant 0x78. +0x38 + pub(crate) unk_38: u32, + + /// +0x3c/+0x40 + pub(crate) cpms_window_size: u32, + pub(crate) cpms_tfca_size: u32, + pub(crate) unk_44: u32, + + /// Kick channel QoS pair. +0x48/+0x4c + pub(crate) kick_channel_qos_b: u32, + pub(crate) kick_channel_qos_a: u32, + + /// G13 `unk_54`/`unk_56`/`unk_58` moved -0x14. +0x50..+0x54 (inferred) + pub(crate) unk_50: u16, + pub(crate) unk_52: u16, + pub(crate) unk_54: u16, + + /// G13 `unk_5a`/`unk_5e`/`unk_62` moved -0x14 (unaligned). +0x56..+0x5e + pub(crate) unk_56: U32, + pub(crate) unk_5a: U32, + pub(crate) cswitch_timer_multiplier: U32, + + /// +0x62 (G13 `unk_66`, tentative) + pub(crate) cdm_cs_mode_change: U32, + pub(crate) unk_66: Array<0xa, u8>, + + /// Command submission enable; must be non-zero to submit work. +0x70 + pub(crate) command_submission_enabled: u32, + + /// Pending-submission counter (G13 `pending_submissions`). +0x74 + pub(crate) pending_submissions: AtomicU32, + + /// +0x78..+0x80 + pub(crate) power_if_target: Array<3, u32>, + + /// +0x84..+0x8c + pub(crate) controller_override_0: u32, + pub(crate) controller_override_1: u32, + pub(crate) perf_state_cap: u32, + + /// Perf-controller block. +0x90..+0xc7 + pub(crate) perf_controller: Array<0x38, u8>, + + /// +0xc8/+0xcc + pub(crate) deadline_control_effort: u32, + pub(crate) deadline_controller_override: u32, + pub(crate) unk_d0: Array<0xa0, u8>, + + /// DPE (dynamic power estimation) block. +0x170..+0x69f + pub(crate) dpe: Array<0x530, u8>, + + /// +0x6a0 + pub(crate) dpe_sl_config: U64, + pub(crate) unk_6a8: Array<0x100, u8>, + + /// Smart idle-off config. +0x7a8 + pub(crate) smart_idle_off_cfg: Array<10, u32>, + + /// +0x7d0..+0x7dc + pub(crate) ut_engagement: u32, + pub(crate) clvr_engagement: u32, + pub(crate) keepalive_perf_threshold_rd: u32, + pub(crate) keepalive_off_threshold_rd: u32, + pub(crate) unk_7e0: u32, + + /// Register overrides. +0x7e4 + pub(crate) reg_overrides: Array<16, RegisterOverride>, + + /// +0x964 + pub(crate) reg_override_count: u32, + pub(crate) unk_968: u32, + + /// Display power-management params. +0x96c/+0x974 + pub(crate) dpm_param_a: U64, + pub(crate) dpm_param_b: U64, + + /// +0x97c..+0x984 + pub(crate) progress_check_interval_3d: u32, + pub(crate) progress_check_interval_ta: u32, + pub(crate) progress_check_interval_cl: u32, + + /// +0x988..+0x99c: the G13 `unk_1102c_0`..`unk_1102c` group (inferred) + pub(crate) unk_1102c_0: u32, + pub(crate) unk_1102c_4: u32, + pub(crate) unk_1102c_8: u32, + pub(crate) unk_1102c_c: u32, + pub(crate) unk_1102c_10: u32, + pub(crate) unk_1102c: u32, + + /// +0x9a0..+0x9a8 + pub(crate) idle_off_delay_ms: AtomicU32, + pub(crate) fender_idle_off_delay_ms: u32, + pub(crate) fw_early_wake_timeout_ms: u32, + + /// +0x9ac + pub(crate) gvdm_timer_interval: u32, + + /// +0x9b0/+0x9b4 + pub(crate) cl_context_switch_timeout_ms: u32, + pub(crate) cl_kill_timeout_ms: u32, + pub(crate) unk_9b8: u32, + + /// +0x9bc + pub(crate) cdm_backoff_timeout: u8, + pub(crate) unk_9bd: Array<0x4, u8>, + + /// +0x9c1/+0x9c3 + pub(crate) fwutil_default_fab_pstate: Array<0x2, u8>, + pub(crate) fwutil_timer_period: u8, + pub(crate) unk_9c4: Array<0x19, u8>, + + /// Unaligned. +0x9dd + pub(crate) mtr_sensor_ptd_override_mask: U64, + pub(crate) unk_9e5: Array<0x400, u8>, + + /// Read by the host when it creates command queues. +0xde5 + pub(crate) unk_de5: U32, + + /// Keep-alive overrides/thresholds. +0xde9..+0xdf5 + pub(crate) gpu_keepalive_override: U32, + pub(crate) gfxc_keepalive_override: U32, + pub(crate) keepalive_perf_threshold: U32, + pub(crate) keepalive_off_threshold: U32, + + /// Soft-fault settings (possibly G13 `fault_control`). +0xdf9 + pub(crate) soft_fault_settings: U32, + pub(crate) unk_dfd: Array<0x3, u8>, + + /// The G13 pending-stamp table is not in the 0xe00-byte G15 Globals. This + /// zero-length field keeps `GpuManager::mark_pending_events()` compiling as a no-op. + /// TODO: find the G15 pending-stamp table (possibly in P0 0x0008..0x4017). + pub(crate) pending_stamps: Array<0, PendingStamp>, + } + default_zeroed!(GlobalsG15V14_8_3); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct UatLevelInfo { + pub(crate) unk_3: u8, + pub(crate) unk_1: u8, + pub(crate) unk_2: u8, + pub(crate) index_shift: u8, + pub(crate) num_entries: u16, + pub(crate) unk_4: u16, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) index_mask: U64, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitDataG15V14_8_3<'a> { + pub(crate) ver_info: Array<0x4, u16>, + + pub(crate) unk_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) runtime_pointers: GpuPointer<'a, super::RuntimePointersG15V14_8_3>, + pub(crate) globals: GpuPointer<'a, super::GlobalsG15V14_8_3>, + + /// G15: no FwStatus pointer; u32 0 then u32 1 (possibly a host-mapped FW allocations + /// flag). +0x28/+0x2c + pub(crate) unk_28: u32, + pub(crate) unk_2c: u32, + // 0x30..0x97 on G15 is a 0x68-byte copy of the UAT pmap descriptor (+0x18..+0x7f). Its + // first 0x64 bytes have the G13 layout (3 levels, 0x20-byte records). + pub(crate) uat_page_size: u16, + pub(crate) uat_page_bits: u8, + pub(crate) uat_num_levels: u8, + pub(crate) uat_level_info: Array<0x3, UatLevelInfo>, + + + /// G15: last 4 bytes of the pmap descriptor copy (descriptor +0x7c). + /// TODO: meaning unknown; left 0. + pub(crate) unk_94: u32, + pub(crate) __pad_98: Pad<0x10>, + + /// P2 debug block (0x20). +0xa8 + pub(crate) debug_block: GpuPointer<'a, super::DebugBlock>, + + /// P0 status block (0xc3d0). +0xb0 + pub(crate) status_block: GpuPointer<'a, super::StatusBlock>, + + /// P1 power-controller block (0x238). +0xb8 + pub(crate) power_ctl_block: GpuPointer<'a, super::PowerCtlBlock>, + } +} + +#[derive(Debug)] +pub(crate) struct ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, +} + +impl ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + raw::ChannelRing { + state: Some(self.state.weak_pointer()), + ring: Some(self.ring.weak_pointer()), + } + } +} + +trivial_gpustruct!(FwStatus); +trivial_gpustruct!(StatusBlock); +trivial_gpustruct!(PowerCtlBlock); +trivial_gpustruct!(DebugBlock); +trivial_gpustruct!(GpuGlobalStatsVtx); +trivial_gpustruct!(GpuGlobalStatsFragG15V14_8_3); +trivial_gpustruct!(GpuStatsComp); + +trivial_gpustruct!(HwDataAG15V14_8_3); + +trivial_gpustruct!(HwDataBG15V14_8_3); + +#[derive(Debug)] +pub(crate) struct StatsG15V14_8_3 { + pub(crate) vtx: GpuObject, + pub(crate) frag: GpuObject, + pub(crate) comp: GpuObject, +} + +#[derive(Debug)] +pub(crate) struct RuntimePointersG15V14_8_3 { + pub(crate) stats: StatsG15V14_8_3, + + pub(crate) hwdata_a: GpuObject, + pub(crate) hwdata_b: GpuObject, + + // G15 group-A regions referenced from RuntimePointers + pub(crate) brn_table: GpuArray, + pub(crate) unkptr_e7: GpuArray, + pub(crate) unkptr_e5: GpuArray, + pub(crate) uma_pool_desc: GpuArray, + + pub(crate) buffer_mgr_ctl: gem::ObjectRef, + pub(crate) buffer_mgr_ctl_low_mapping: Option, + pub(crate) buffer_mgr_ctl_high_mapping: Option, +} + +impl GpuStruct for RuntimePointersG15V14_8_3 { + type Raw<'a> = raw::RuntimePointersG15V14_8_3<'a>; +} + +trivial_gpustruct!(GlobalsG15V14_8_3); + +#[derive(Debug)] +pub(crate) struct InitDataG15V14_8_3 { + pub(crate) unk_buf: GpuArray, + pub(crate) runtime_pointers: GpuObject, + pub(crate) globals: GpuObject, + + /// G15: the P0 status block, which embeds the FwStatus FWCtl ring and halt flags. It keeps + /// the `fw_status` name so the FwStatus users see the same fields. + pub(crate) fw_status: GpuObject, + pub(crate) power_ctl_block: GpuObject, + pub(crate) debug_block: GpuObject, +} + +impl GpuStruct for InitDataG15V14_8_3 { + type Raw<'a> = raw::InitDataG15V14_8_3<'a>; +} + +// G15 / 14.8.3 firmware structure layout checks: the sizes and offsets the firmware expects. +mod g15_layout { + use super::channels; + use super::raw::*; + use core::mem::{offset_of, size_of}; + use kernel::static_assert; + + // InitData, 0xc0 bytes used + static_assert!(size_of::>() == 0xc0); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, unk_buf) == 0x08); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, runtime_pointers) == 0x18); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, globals) == 0x20); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, unk_2c) == 0x2c); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, uat_page_size) == 0x30); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, debug_block) == 0xa8); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, status_block) == 0xb0); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, power_ctl_block) == 0xb8); + + // host->FW ring descriptor + static_assert!(size_of::() == 0x20); + static_assert!(size_of::() == 0x60); + + // RuntimePointers: 0x490 bytes, packed + static_assert!(size_of::>() == 0x490); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, hwdata_b) == 0x000); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, brn_table) == 0x008); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_e7) == 0x010); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, pipes) == 0x018); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, device_control) == 0x198); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, event) == 0x1b8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, fw_log) == 0x1c8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, ktrace) == 0x1d8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats) == 0x1e8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, fwlog_buf) == 0x1f8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_230) == 0x230); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_vtx) == 0x234); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_frag) == 0x23c); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_comp) == 0x244); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_e5) == 0x24c); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_2a8) == 0x2a8); + static_assert!( + offset_of!(RuntimePointersG15V14_8_3<'static>, buffer_mgr_ctl_gpu_addr) == 0x2b0 + ); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, buffer_mgr_ctl_fw_addr) == 0x2b8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, uma_pool_desc_gpu_addr) == 0x2c0); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, uma_pool_desc_fw_addr) == 0x2c8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_2d0) == 0x2d0); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, cswitch_hist_ctl) == 0x3b0); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, hwdata_a) == 0x441); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_449) == 0x449); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_46d) == 0x46d); + + // Globals: 0xe00 bytes, packed + static_assert!(size_of::() == 0xe00); + static_assert!(size_of::() == 0x18); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_28) == 0x28); + static_assert!(offset_of!(GlobalsG15V14_8_3, debug) == crate::m3_globals_debug::OFFSET); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_38) == 0x38); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_50) == 0x50); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_56) == 0x56); + static_assert!(offset_of!(GlobalsG15V14_8_3, cswitch_timer_multiplier) == 0x5e); + static_assert!(offset_of!(GlobalsG15V14_8_3, command_submission_enabled) == 0x70); + static_assert!(offset_of!(GlobalsG15V14_8_3, pending_submissions) == 0x74); + static_assert!(offset_of!(GlobalsG15V14_8_3, perf_controller) == 0x90); + static_assert!(offset_of!(GlobalsG15V14_8_3, deadline_control_effort) == 0xc8); + static_assert!(offset_of!(GlobalsG15V14_8_3, dpe) == 0x170); + static_assert!(offset_of!(GlobalsG15V14_8_3, dpe_sl_config) == 0x6a0); + static_assert!(offset_of!(GlobalsG15V14_8_3, smart_idle_off_cfg) == 0x7a8); + static_assert!(offset_of!(GlobalsG15V14_8_3, ut_engagement) == 0x7d0); + static_assert!(offset_of!(GlobalsG15V14_8_3, reg_overrides) == 0x7e4); + static_assert!(offset_of!(GlobalsG15V14_8_3, reg_override_count) == 0x964); + static_assert!(offset_of!(GlobalsG15V14_8_3, dpm_param_a) == 0x96c); + static_assert!(offset_of!(GlobalsG15V14_8_3, progress_check_interval_3d) == 0x97c); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_1102c_0) == 0x988); + static_assert!(offset_of!(GlobalsG15V14_8_3, idle_off_delay_ms) == 0x9a0); + static_assert!(offset_of!(GlobalsG15V14_8_3, fender_idle_off_delay_ms) == 0x9a4); + static_assert!(offset_of!(GlobalsG15V14_8_3, fw_early_wake_timeout_ms) == 0x9a8); + static_assert!(offset_of!(GlobalsG15V14_8_3, cl_context_switch_timeout_ms) == 0x9b0); + static_assert!(offset_of!(GlobalsG15V14_8_3, cdm_backoff_timeout) == 0x9bc); + static_assert!(offset_of!(GlobalsG15V14_8_3, fwutil_default_fab_pstate) == 0x9c1); + static_assert!(offset_of!(GlobalsG15V14_8_3, mtr_sensor_ptd_override_mask) == 0x9dd); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_de5) == 0xde5); + static_assert!(offset_of!(GlobalsG15V14_8_3, soft_fault_settings) == 0xdf9); + + // P0 status block + static_assert!(size_of::() == 0xc3d0); + static_assert!(offset_of!(StatusBlock, fw_recovery_info_a) == 0x40a0); + static_assert!(offset_of!(StatusBlock, recovery_packet_info) == 0x44a0); + static_assert!(offset_of!(StatusBlock, unk_44c0) == 0x44c0); + static_assert!(offset_of!(StatusBlock, unk_4558) == 0x4558); + static_assert!(offset_of!(StatusBlock, fwctl_channel) == 0x4568); + static_assert!(offset_of!(StatusBlock, flags) == 0x4580); + static_assert!(offset_of!(StatusBlock, flags.halted) == 0x4590); + static_assert!(offset_of!(StatusBlock, flags.resume) == 0x45a0); + static_assert!(offset_of!(StatusBlock, flags.unk_40) == 0x45b0); + static_assert!(offset_of!(StatusBlock, flags.sched_callbacks) == 0x45c4); + static_assert!(offset_of!(StatusBlock, qos_mode) == 0xc3c8); + + // P1 power-controller block + static_assert!(size_of::() == 0x238); + static_assert!(offset_of!(PowerCtlBlock, power_zone_count) == 0x0c); + static_assert!(offset_of!(PowerCtlBlock, power_zones) == 0x24); + static_assert!(offset_of!(PowerCtlBlock, clvr_loop_override_a) == 0x60); + static_assert!(offset_of!(PowerCtlBlock, lifetime_servo_temp_a) == 0xa4); + static_assert!(offset_of!(PowerCtlBlock, fast_die0_release_temp) == 0xac); + static_assert!(offset_of!(PowerCtlBlock, ppm_words) == 0xc8); + static_assert!(offset_of!(PowerCtlBlock, unk_dc) == 0xdc); + static_assert!(offset_of!(PowerCtlBlock, se_controller_override) == 0xe8); + static_assert!(offset_of!(PowerCtlBlock, se_engagement) == 0xec); + static_assert!(offset_of!(PowerCtlBlock, dpe_leak_cfg_a) == 0x158); + static_assert!(offset_of!(PowerCtlBlock, dpe_leak_cfg_b) == 0x1a8); + static_assert!(offset_of!(PowerCtlBlock, clpc_shared_a) == 0x1b4); + static_assert!(offset_of!(PowerCtlBlock, perf_ctrl_override_active) == 0x1d0); + static_assert!(offset_of!(PowerCtlBlock, consistent_perf_state) == 0x1d1); + static_assert!(offset_of!(PowerCtlBlock, accumulated_energy) == 0x1d8); + static_assert!(offset_of!(PowerCtlBlock, dsid_cfg_a) == 0x1e8); + static_assert!(offset_of!(PowerCtlBlock, dsid_cfg_b) == 0x218); + static_assert!(offset_of!(PowerCtlBlock, submit_in_progress_gate) == 0x230); + + // P2 debug block + static_assert!(size_of::() == 0x20); + static_assert!(offset_of!(DebugBlock, system_sleep_in_progress) == 0x08); + static_assert!(offset_of!(DebugBlock, fw_word_10) == 0x10); + static_assert!(offset_of!(DebugBlock, init_state) == 0x14); + + // InitSeq record + static_assert!(size_of::() == 0x18); + static_assert!(offset_of!(InitSeqRecord, kind) == 0x10); + + // HwDataB: 0x1868 bytes + static_assert!(size_of::() == 0x1868); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_30) == 0x20); + static_assert!(offset_of!(HwDataBG15V14_8_3, timestamp_area_base) == 0x28); + static_assert!(offset_of!(HwDataBG15V14_8_3, yuv_matrices) == 0x38); + static_assert!(offset_of!(HwDataBG15V14_8_3, pad_1c8) == 0x638); + static_assert!(offset_of!(HwDataBG15V14_8_3, io_mappings) == 0x640); + static_assert!(offset_of!(HwDataBG15V14_8_3, sgx_sram_ptr) == 0xa20); + static_assert!(offset_of!(HwDataBG15V14_8_3, chip_id) == 0xa28); + static_assert!(offset_of!(HwDataBG15V14_8_3, base_clock_khz) == 0xa68); + static_assert!(offset_of!(HwDataBG15V14_8_3, power_sample_period) == 0xa6c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_a9c) == 0xa9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_ac8) == 0xac8); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_b24) == 0xb24); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_554) == 0xb40); + static_assert!(offset_of!(HwDataBG15V14_8_3, uat_ttb_base) == 0xb44); + static_assert!(offset_of!(HwDataBG15V14_8_3, gpu_core_id) == 0xb4c); + static_assert!(offset_of!(HwDataBG15V14_8_3, num_cores) == 0xb54); + static_assert!(offset_of!(HwDataBG15V14_8_3, max_pstate) == 0xb58); + static_assert!(offset_of!(HwDataBG15V14_8_3, frequencies) == 0xb5c); + static_assert!(offset_of!(HwDataBG15V14_8_3, voltages) == 0xb9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, voltages_sram) == 0xd9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, frequencies_2) == 0xf9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, sram_k) == 0xfdc); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_9f4) == 0x101c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_arr_0) == 0x10dc); + static_assert!(offset_of!(HwDataBG15V14_8_3, aux_ps) == 0x115c); + static_assert!(offset_of!(HwDataBG15V14_8_3, aux_ps_3) == 0x134c); + static_assert!(size_of::() == 0x444); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_179c) == 0x179c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unit_mask_a) == 0x17c0); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_17e8) == 0x17e8); + static_assert!(offset_of!(HwDataBG15V14_8_3, timer_offset) == 0x17ec); + static_assert!(offset_of!(HwDataBG15V14_8_3, all_ones_masks) == 0x1820); + static_assert!(offset_of!(HwDataBG15V14_8_3, fstp_override) == 0x1850); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_1860) == 0x1860); + static_assert!(size_of::() == 0x20); + + // HwDataA: 0x4360 bytes, G13 13.5 layout, +0x10 from 0x1288 + static_assert!(size_of::() == 0x4360); + static_assert!(offset_of!(HwDataAG15V14_8_3, clocks_per_period) == 0x4); + static_assert!(offset_of!(HwDataAG15V14_8_3, pwr_status) == 0x10); + static_assert!(offset_of!(HwDataAG15V14_8_3, actual_pstate) == 0x2c); + static_assert!(offset_of!(HwDataAG15V14_8_3, min_pstate_scaled) == 0x54); + static_assert!(offset_of!(HwDataAG15V14_8_3, sram_k) == 0x80); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_64c) == 0x658); + static_assert!(offset_of!(HwDataAG15V14_8_3, pwr_filter_a_neg) == 0x664); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_power_2) == 0x6a4); + static_assert!(offset_of!(HwDataAG15V14_8_3, perf_tgt_utilization) == 0x788); + static_assert!(offset_of!(HwDataAG15V14_8_3, perf_filter_drop_threshold) == 0x7a8); + static_assert!(offset_of!(HwDataAG15V14_8_3, perf_tgt_utilization_2) == 0x7ec); + static_assert!(offset_of!(HwDataAG15V14_8_3, min_pstate_scaled_4) == 0x860); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask) == 0x8ac); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_release_temp_cc) == 0x8b4); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_ki_dt) == 0x8d8); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_kp) == 0x8e8); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_prop_tgt_delta) == 0x900); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_10) == 0x938); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_coef_a1) == 0x970); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_11) == 0xbe0); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_c2c) == 0xe80); + static_assert!(offset_of!(HwDataAG15V14_8_3, power_zone_count) == 0xe84); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_target_filter_a_neg) == 0xe9c); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_filter_tc_periods_x4) == 0xf7c); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_kp) == 0xfa8); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_filter_tc_ms) == 0xfd0); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_14) == 0xff8); + // The builder relies on this base for the G15 +0x10f4/+0x11d0 stores + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_e10_0) == 0x10b4); + static_assert!(size_of::() == 0x1b8); + static_assert!(offset_of!(HwDataA140Extra, unk_40) == 0x40); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_filter_a_neg) == 0x4c); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_kp) == 0x6c); + static_assert!(offset_of!(HwDataA140Extra, max_pstate_scaled_1) == 0x7c); + static_assert!(offset_of!(HwDataA140Extra, se_target) == 0x8c); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_filter_time_constant_clks) == 0x9c); + static_assert!(offset_of!(HwDataA140Extra, unk_b0) == 0xb0); + static_assert!(offset_of!(HwDataA140Extra, unk_d8) == 0xd8); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_reset_criteria) == 0xf0); + static_assert!(offset_of!(HwDataA140Extra, unk_11c) == 0x11c); + static_assert!(offset_of!(HwDataA140Extra, max_pstate_scaled_2) == 0x130); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask_2) == 0x1288); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_e28) == 0x1294); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask_alt) == 0x1a98); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_3640) == 0x3aa4); + static_assert!(offset_of!(HwDataAG15V14_8_3, init_timestamp) == 0x424c); + static_assert!(offset_of!(HwDataAG15V14_8_3, cluster_tables) == 0x42a4); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_4340) == 0x4340); + static_assert!(offset_of!(HwDataAG15V14_8_3, gpu_keepalive_mode) == 0x4358); + + // E2 StatsFrag: 0x1248 bytes, -1 fields at +0xc18/+0xc30 + static_assert!(size_of::() == 0x1248); + static_assert!( + offset_of!(GpuGlobalStatsFragG15V14_8_3, stats) + + offset_of!(GpuStatsFragG15V14_8_3, cur_stamp_id) + == 0xc18 + ); + static_assert!( + offset_of!(GpuGlobalStatsFragG15V14_8_3, stats) + + offset_of!(GpuStatsFragG15V14_8_3, unk_id) + == 0xc30 + ); + + // G13/G14X 13.5 sizes must not change (regression guard for the per-field gating). + // RuntimePointers: gpu_scratch at 0x280 (0x270 on G14X) + RuntimeScratch; note that these + // are 0x20 more than ver_info word 0 (0x6ba0/0xb390). The others are the current G13/G14X + // sizes. + static_assert!(size_of::>() == 0x6bc0); + static_assert!(size_of::>() == 0xb3b0); + static_assert!(size_of::() == 0x12394); + static_assert!(size_of::() == 0x1715c); + static_assert!(size_of::() == 0x421c); + static_assert!(size_of::() == 0x6c34); + static_assert!(size_of::() == 0x1884); + static_assert!(size_of::() == 0x1884); + static_assert!(size_of::>() == 0xbc); + static_assert!(size_of::>() == 0xbc); + static_assert!(size_of::>() == 0xb4); + static_assert!(size_of::() == 0x30); + static_assert!(size_of::() == 0x1124); + static_assert!(size_of::() == 0x1a34); + static_assert!(size_of::>() == 0x30); +} diff --git a/drivers/gpu/drm/asahi/m3/hw.rs b/drivers/gpu/drm/asahi/m3/hw.rs new file mode 100644 index 0000000000000..c90156bd1244b --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/hw.rs @@ -0,0 +1,679 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-SoC hardware configuration structures +//! +//! This module contains the definitions used to store per-GPU and per-SoC configuration data. + +use crate::fw::types::*; +pub(crate) use crate::identity::{GpuGen, GpuHalGeneration, GpuVariant}; +use kernel::c_str; +use kernel::prelude::*; + +const MAX_POWERZONES: usize = 5; + +pub(crate) mod agx3; +pub(crate) mod t600x; +pub(crate) mod t602x; +pub(crate) mod t6030; +pub(crate) mod t6031; +pub(crate) mod t8103; +pub(crate) mod t8112; +pub(crate) mod t8122; +pub(crate) mod t8140; + +/// GPU revision enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevision { + A0 = 0x00, + A1 = 0x01, + B0 = 0x10, + B1 = 0x11, + /// The chip revision T6031 (M3 Max) reports (/arm-io chip-revision 0x12). Only the T6031 + /// table names it; no identification path maps a register read to it yet. + #[allow(dead_code)] + B2 = 0x12, + C0 = 0x20, + C1 = 0x21, +} + +/// GPU core type enumeration. Note: Part of the firmware ABI. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum GpuCore { + // Unknown = 0, + // G5P = 1, + // G5G = 2, + // G9P = 3, + // G9G = 4, + // G10P = 5, + // G11P = 6, + // G11M = 7, + // G11G = 8, + // G12P = 9, + // G13P = 10, + G13G = 11, + G13S = 12, + G13C = 13, + // G14P = 14, + G14G = 15, + G14S = 16, + G14C = 17, + G14D = 18, // Split out, unlike G13D + // G15 firmware core type IDs (G15S = 0x17). + // G15M = 19, + // G15P_AGX2 = 20, + // G15P = 21, + G15G = 22, + G15S = 23, + /// Not confirmed on hardware yet; only the T6031 table uses it. + #[allow(dead_code)] + G15C = 24, +} + +/// GPU revision ID. Note: Part of the firmware ABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevisionID { + // Unknown = 0, + A0 = 1, + A1 = 2, + B0 = 3, + B1 = 4, + C0 = 5, + C1 = 6, +} + +/// A single performance state of the GPU. +#[derive(Debug)] +pub(crate) struct PState { + /// Voltage in millivolts, per GPU cluster. + pub(crate) volt_mv: KVec, + /// Frequency in hertz. + pub(crate) freq_hz: u32, + /// Maximum power consumption of the GPU at this pstate, in milliwatts. + pub(crate) pwr_mw: u32, +} + +impl PState { + pub(crate) fn max_volt_mv(&self) -> u32 { + *self.volt_mv.iter().max().expect("No voltages") + } +} + +/// A power zone definition (we have no idea what this is but Apple puts them in the DT). +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct PowerZone { + pub(crate) target: u32, + pub(crate) target_offset: u32, + pub(crate) filter_tc: u32, +} + +/// An MMIO mapping used by the firmware. +#[derive(Debug, Copy, Clone)] +pub(crate) struct IOMapping { + /// Base physical address of the mapping. + pub(crate) base: usize, + /// Whether this mapping should be replicated to all dies + pub(crate) per_die: bool, + /// Number of mappings. + pub(crate) count: usize, + /// Size of one mapping. + pub(crate) size: usize, + /// Stride between mappings. + pub(crate) stride: usize, + /// Whether the mapping should be writable. + pub(crate) writable: bool, +} + +impl IOMapping { + /// Convenience constructor for a new IOMapping. + pub(crate) const fn new( + base: usize, + per_die: bool, + count: usize, + size: usize, + stride: usize, + writable: bool, + ) -> IOMapping { + IOMapping { + base, + per_die, + count, + size, + stride, + writable, + } + } +} + +/// Unknown HwConfigA fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigA { + pub(crate) unk_87c: i32, + pub(crate) unk_8cc: u32, + pub(crate) unk_e24: u32, +} + +/// Unknown HwConfigB fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigB { + pub(crate) unk_454: u32, + pub(crate) unk_4e0: u64, + pub(crate) unk_534: u32, + pub(crate) unk_ab8: u32, + pub(crate) unk_abc: u32, + pub(crate) unk_b30: u32, +} + +/// Render command configs that vary from SoC to SoC. +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwRenderConfig { + /// Vertex/tiling-related configuration register (lsb: disable clustering) + pub(crate) tiling_control: u32, +} + +#[derive(Debug)] +pub(crate) struct HwConfigShared2Curves { + pub(crate) t1_coef: u32, + pub(crate) t2: &'static [i16], + pub(crate) t3_coefs: &'static [u32], + pub(crate) t3_scales: &'static [u32], +} + +/// Static hardware clustering configuration for multi-cluster SoCs. +#[derive(Debug)] +pub(crate) struct HwClusteringConfig { + pub(crate) meta1_blocksize: usize, + pub(crate) meta2_size: usize, + pub(crate) meta3_size: usize, + pub(crate) meta4_size: usize, + pub(crate) max_splits: usize, +} + +/// Static hardware configuration for a given SoC model. +#[derive(Debug)] +pub(crate) struct HwConfig { + /// Chip ID in hex format (e.g. 0x8103 for t8103). + pub(crate) chip_id: u32, + /// GPU generation. + pub(crate) gpu_gen: GpuGen, + /// GPU variant type. + pub(crate) gpu_variant: GpuVariant, + /// GPU core type ID (as known by the firmware). + /// + /// This is a firmware-ABI value (renumbered per firmware build). It is + /// `None` where no firmware GpuCore id has been recovered (AGX3 parts): + /// consumers must fail closed rather than substitute a number. + pub(crate) gpu_core: Option, + + /// Base clock used used for timekeeping. + pub(crate) base_clock_hz: u32, + /// Output address space for the UAT on this SoC. + pub(crate) uat_oas: u32, + /// Input-address bits translated by each UAT root. + pub(crate) uat_ias: u8, + /// Number of dies on this SoC. + pub(crate) num_dies: u32, + /// Maximum number of clusters on this SoC. + pub(crate) max_num_clusters: u32, + /// Maximum number of cores per cluster for this GPU. + pub(crate) max_num_cores: u32, + /// Maximum number of frags per cluster for this GPU. + pub(crate) max_num_frags: u32, + /// Maximum number of GPs per cluster for this GPU. + pub(crate) max_num_gps: u32, + + /// Required size of the first preemption buffer. + pub(crate) preempt1_size: usize, + /// Required size of the second preemption buffer. + pub(crate) preempt2_size: usize, + /// Required size of the third preemption buffer. + pub(crate) preempt3_size: usize, + + /// Required size of the compute preemption buffer. + pub(crate) compute_preempt1_size: usize, + + pub(crate) clustering: Option, + + /// Rendering-relevant configuration. + pub(crate) render: HwRenderConfig, + + /// Misc HWDataA field values. + pub(crate) da: HwConfigA, + /// Misc HWDataB field values. + pub(crate) db: HwConfigB, + /// HwDataShared1.table. + pub(crate) shared1_tab: &'static [i32], + /// HwDataShared1.unk_a4. + pub(crate) shared1_a4: u32, + /// HwDataShared2.table. + pub(crate) shared2_tab: &'static [i32], + /// HwDataShared2.unk_508. + pub(crate) shared2_unk_508: u32, + /// HwDataShared2.unk_508. + pub(crate) shared2_curves: Option, + + /// HwDataShared3.unk_8. + pub(crate) shared3_unk: u32, + /// HwDataShared3.table. + pub(crate) shared3_tab: &'static [u32], + + /// Globals.idle_off_standby_timer. + pub(crate) idle_off_standby_timer_default: u32, + /// Globals.unk_hws2_4. + pub(crate) unk_hws2_4: Option<[F32; 8]>, + /// Globals.unk_hws2_24. + pub(crate) unk_hws2_24: u32, + /// Globals.unk_54 + pub(crate) global_unk_54: u16, + + /// Constant related to SRAM voltages. + pub(crate) sram_k: F32, + /// Unknown per-cluster coefficients 1. + pub(crate) unk_coef_a: &'static [&'static [F32]], + /// Unknown per-cluster coefficients 2. + pub(crate) unk_coef_b: &'static [&'static [F32]], + /// Unknown table in Global struct. + pub(crate) global_tab: Option<&'static [u8]>, + /// Whether this GPU has CS/AFR performance states + pub(crate) has_csafr: bool, + + /// Temperature sensor list (8 bits per sensor). + pub(crate) fast_sensor_mask: [u64; 2], + /// Temperature sensor list (alternate). + pub(crate) fast_sensor_mask_alt: [u64; 2], + /// Temperature sensor present bitmask. + pub(crate) fast_die0_sensor_present: u32, + /// Required MMIO mappings for this GPU/firmware. + pub(crate) io_mappings: &'static [Option], + /// SRAM base + pub(crate) sram_base: Option, + /// SRAM size + pub(crate) sram_size: Option, +} + +/// Dynamic (fetched from hardware/DT) configuration. +#[derive(Debug)] +pub(crate) struct DynConfig { + /// Base physical address of the UAT TTB (from DT reserved memory region). + pub(crate) uat_ttb_base: u64, + /// GPU ID configuration read from hardware. + pub(crate) id: GpuIdConfig, + /// Power calibration configuration for this specific chip/device. + pub(crate) pwr: PwrConfig, + /// Firmware version. + #[allow(dead_code)] + pub(crate) firmware_version: KVec, + + pub(crate) hw_data_a: KVVec, + pub(crate) hw_data_b: KVVec, + pub(crate) hw_globals: KVVec, +} + +/// Specific GPU ID configuration fetched from SGX MMIO registers. +#[derive(Debug)] +pub(crate) struct GpuIdConfig { + /// GPU generation (should match static config). + pub(crate) gpu_gen: GpuGen, + /// GPU variant type (should match static config). + pub(crate) gpu_variant: GpuVariant, + /// Unified shader core generation. + pub(crate) usc_generation: u32, + /// Apple GPU HAL generation (0 for the pre-numbered generations). + pub(crate) gpu_hal_generation: GpuHalGeneration, + /// GPU silicon revision. + pub(crate) gpu_rev: GpuRevision, + /// GPU silicon revision ID (firmware enum). + pub(crate) gpu_rev_id: GpuRevisionID, + /// Number of GPU dies read from the ID register. + pub(crate) num_dies: u32, + /// Total number of GPU clusters. + pub(crate) num_clusters: u32, + /// Maximum number of GPU cores per cluster. + pub(crate) num_cores: u32, + /// Number of frags per cluster. + pub(crate) num_frags: u32, + /// Number of GPs per cluster. + pub(crate) num_gps: u32, + /// Total number of active cores for the whole GPU. + pub(crate) total_active_cores: u32, + /// Mask of active cores per cluster. + pub(crate) core_masks: KVec, + /// Packed mask of all active cores. + pub(crate) core_masks_packed: KVec, + +} + +/// Configurable CS/AFR GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct CsAfrPwrConfig { + /// GPU CS performance state list. + pub(crate) perf_states_cs: KVec, + /// GPU AFR performance state list. + pub(crate) perf_states_afr: KVec, + + /// CS leakage coefficient per die. + pub(crate) leak_coef_cs: KVec, + /// AFR leakage coefficient per die. + pub(crate) leak_coef_afr: KVec, + + /// Minimum voltage for the CS/AFR SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, +} + +/// Configurable GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct PwrConfig { + /// GPU performance state list. + pub(crate) perf_states: KVec, + /// GPU power zone list. + pub(crate) power_zones: KVec, + + /// Core leakage coefficient per cluster. + pub(crate) core_leak_coef: KVec, + /// SRAM leakage coefficient per cluster. + pub(crate) sram_leak_coef: KVec, + + pub(crate) csafr: Option, + + /// Maximum total power of the GPU in milliwatts. + pub(crate) max_power_mw: u32, + /// Maximum frequency of the GPU in megahertz. + pub(crate) max_freq_mhz: u32, + + /// Minimum performance state to start at. + pub(crate) perf_base_pstate: u32, + /// Maximum enabled performance state. + pub(crate) perf_max_pstate: u32, + + /// Minimum voltage for the SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, + + // Most of these fields are just named after Apple ADT property names and we don't fully + // understand them. They configure various power-related PID loops and filters. + /// Average power filter time constant in milliseconds. + pub(crate) avg_power_filter_tc_ms: u32, + /// Average power filter PID integral gain? + pub(crate) avg_power_ki_only: F32, + /// Average power filter PID proportional gain? + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + /// Average power target filter time constant in periods. + pub(crate) avg_power_target_filter_tc: u32, + /// "Fast die0" (temperature?) PID integral gain. + pub(crate) fast_die0_integral_gain: F32, + /// "Fast die0" (temperature?) PID proportional gain. + pub(crate) fast_die0_proportional_gain: F32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_release_temp: u32, + /// Delay from the fender (?) becoming idle to powerdown + pub(crate) fender_idle_off_delay_ms: u32, + /// Timeout from firmware early wake to sleep if no work was submitted (?) + pub(crate) fw_early_wake_timeout_ms: u32, + /// Delay from the GPU becoming idle to powerdown + pub(crate) idle_off_delay_ms: u32, + /// Related to the above? + pub(crate) idle_off_standby_timer: u32, + /// Percent? + pub(crate) perf_boost_ce_step: u32, + /// Minimum utilization before performance state is increased in %. + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_filter_drop_threshold: u32, + /// Performance PID filter time constant? (periods?) + pub(crate) perf_filter_time_constant: u32, + /// Performance PID filter time constant 2? (periods?) + pub(crate) perf_filter_time_constant2: u32, + /// Performance PID integral gain. + pub(crate) perf_integral_gain: F32, + /// Performance PID integral gain 2 (?). + pub(crate) perf_integral_gain2: F32, + pub(crate) perf_integral_min_clamp: u32, + /// Performance PID proportional gain. + pub(crate) perf_proportional_gain: F32, + /// Performance PID proportional gain 2 (?). + pub(crate) perf_proportional_gain2: F32, + pub(crate) perf_reset_iters: u32, + /// Target GPU utilization for the performance controller in %. + pub(crate) perf_tgt_utilization: u32, + /// Power sampling period in milliseconds. + pub(crate) power_sample_period: u32, + /// PPM (?) filter time constant in milliseconds. + pub(crate) ppm_filter_time_constant_ms: u32, + /// PPM (?) filter PID integral gain. + pub(crate) ppm_ki: F32, + /// PPM (?) filter PID proportional gain. + pub(crate) ppm_kp: F32, + /// Power consumption filter time constant (periods?) + pub(crate) pwr_filter_time_constant: u32, + /// Power consumption filter PID integral gain. + pub(crate) pwr_integral_gain: F32, + pub(crate) pwr_integral_min_clamp: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) pwr_proportional_gain: F32, + /// Power sample period in base clocks, used when not an integer number of ms + pub(crate) pwr_sample_period_aic_clks: u32, + + pub(crate) se_engagement_criteria: i32, + pub(crate) se_filter_time_constant: u32, + pub(crate) se_filter_time_constant_1: u32, + pub(crate) se_inactive_threshold: u32, + pub(crate) se_ki: F32, + pub(crate) se_ki_1: F32, + pub(crate) se_kp: F32, + pub(crate) se_kp_1: F32, + pub(crate) se_reset_criteria: u32, + /// Shader-engine controller target (G15 runtime layout). + pub(crate) se_target: u32, +} + +impl PwrConfig { + fn load_opp( + dev: &kernel::device::Device, + name: &CStr, + cfg: &HwConfig, + is_main: bool, + ) -> Result> { + let mut perf_states = KVec::new(); + + let node = dev.of_node().ok_or(EIO)?; + let opps = node.parse_phandle(name, 0).ok_or(EIO)?; + + for opp in opps.children() { + let freq_hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let mut volt_uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; + let pwr_uw: u32 = if is_main { + opp.get_property(c_str!("opp-microwatt"))? + } else { + 0 + }; + + let voltage_count = if is_main { + cfg.max_num_clusters + } else { + cfg.num_dies + }; + + if volt_uv.len() != voltage_count as usize { + dev_err!( + dev, + "Invalid opp-microvolt length (expected {}, got {})\n", + voltage_count, + volt_uv.len() + ); + return Err(EINVAL); + } + + // Round up: a voltage is never published below the device tree's. + volt_uv.iter_mut().for_each(|a| *a = if cfg.gpu_gen == GpuGen::G15 { a.div_ceil(1000) } else { *a / 1000 }); + let volt_mv = volt_uv; + + let pwr_mw = pwr_uw / 1000; + + perf_states.push( + PState { + freq_hz: freq_hz.try_into()?, + volt_mv, + pwr_mw, + }, + GFP_KERNEL, + )?; + } + + if perf_states.is_empty() { + Err(EINVAL) + } else { + Ok(perf_states) + } + } + + /// Load the GPU power configuration from the device tree node of `dev`. + pub(crate) fn load(dev: &kernel::device::Device, cfg: &HwConfig) -> Result { + let perf_states = Self::load_opp(dev, c_str!("operating-points-v2"), cfg, true)?; + let node = dev.of_node().ok_or(EIO)?; + + macro_rules! prop { + ($prop:expr, $default:expr) => {{ + node.get_opt_property(c_str!($prop)) + .map_err(|e| { + dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); + e + })? + .unwrap_or($default) + }}; + ($prop:expr) => {{ + node.get_property(c_str!($prop)).map_err(|e| { + dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); + e + })? + }}; + } + + let pz_data = prop!("apple,power-zones", KVec::new()); + + if pz_data.len() > 3 * MAX_POWERZONES || pz_data.len() % 3 != 0 { + dev_err!(dev, "Invalid apple,power-zones value\n"); + return Err(EINVAL); + } + + let pz_count = pz_data.len() / 3; + let mut power_zones = KVec::new(); + for i in (0..pz_count).step_by(3) { + power_zones.push( + PowerZone { + target: pz_data[i], + target_offset: pz_data[i + 1], + filter_tc: pz_data[i + 2], + }, + GFP_KERNEL, + )?; + } + + let core_leak_coef: KVec = prop!("apple,core-leak-coef"); + let sram_leak_coef: KVec = prop!("apple,sram-leak-coef"); + + if core_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev, "Invalid apple,core-leak-coef\n"); + return Err(EINVAL); + } + if sram_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev, "Invalid apple,sram_leak_coef\n"); + return Err(EINVAL); + } + + let csafr = if cfg.has_csafr { + Some(CsAfrPwrConfig { + perf_states_cs: Self::load_opp(dev, c_str!("apple,cs-opp"), cfg, false)?, + perf_states_afr: Self::load_opp(dev, c_str!("apple,afr-opp"), cfg, false)?, + leak_coef_cs: prop!("apple,cs-leak-coef"), + leak_coef_afr: prop!("apple,afr-leak-coef"), + min_sram_microvolt: prop!("apple,csafr-min-sram-microvolt"), + }) + } else { + None + }; + + let power_sample_period: u32 = prop!("apple,power-sample-period"); + + Ok(PwrConfig { + core_leak_coef, + sram_leak_coef, + + max_power_mw: perf_states.iter().map(|a| a.pwr_mw).max().unwrap(), + max_freq_mhz: perf_states.iter().map(|a| a.freq_hz).max().unwrap() / 1_000_000, + + perf_base_pstate: prop!("apple,perf-base-pstate", 1), + perf_max_pstate: perf_states.len() as u32 - 1, + min_sram_microvolt: if cfg.gpu_gen == GpuGen::G15 { + // SRAM rail floor: 810 mV from the J516S runtime ADT perf-states-sram table. + // Tolerate a DT without the property on G15 only. + prop!("apple,min-sram-microvolt", t6030::MIN_SRAM_MICROVOLT) + } else { + prop!("apple,min-sram-microvolt") + }, + + avg_power_filter_tc_ms: prop!("apple,avg-power-filter-tc-ms"), + avg_power_ki_only: prop!("apple,avg-power-ki-only"), + avg_power_kp: prop!("apple,avg-power-kp"), + avg_power_min_duty_cycle: prop!("apple,avg-power-min-duty-cycle"), + avg_power_target_filter_tc: prop!("apple,avg-power-target-filter-tc"), + fast_die0_integral_gain: prop!("apple,fast-die0-integral-gain"), + fast_die0_proportional_gain: prop!("apple,fast-die0-proportional-gain"), + fast_die0_prop_tgt_delta: prop!("apple,fast-die0-prop-tgt-delta", 0), + fast_die0_release_temp: prop!("apple,fast-die0-release-temp", 80), + fender_idle_off_delay_ms: prop!("apple,fender-idle-off-delay-ms", 40), + fw_early_wake_timeout_ms: prop!("apple,fw-early-wake-timeout-ms", 5), + idle_off_delay_ms: prop!("apple,idle-off-delay-ms", 2), + idle_off_standby_timer: prop!( + "apple,idleoff-standby-timer", + cfg.idle_off_standby_timer_default + ), + perf_boost_ce_step: prop!("apple,perf-boost-ce-step", 25), + perf_boost_min_util: prop!("apple,perf-boost-min-util", 100), + perf_filter_drop_threshold: prop!("apple,perf-filter-drop-threshold"), + perf_filter_time_constant2: prop!("apple,perf-filter-time-constant2"), + perf_filter_time_constant: prop!("apple,perf-filter-time-constant"), + perf_integral_gain2: prop!("apple,perf-integral-gain2"), + perf_integral_gain: prop!("apple,perf-integral-gain", f32!(7.8956833)), + perf_integral_min_clamp: prop!("apple,perf-integral-min-clamp"), + perf_proportional_gain2: prop!("apple,perf-proportional-gain2"), + perf_proportional_gain: prop!("apple,perf-proportional-gain", f32!(14.707963)), + perf_reset_iters: prop!("apple,perf-reset-iters", 6), + perf_tgt_utilization: prop!("apple,perf-tgt-utilization"), + power_sample_period, + ppm_filter_time_constant_ms: prop!("apple,ppm-filter-time-constant-ms"), + ppm_ki: prop!("apple,ppm-ki"), + ppm_kp: prop!("apple,ppm-kp"), + pwr_filter_time_constant: prop!("apple,pwr-filter-time-constant", 313), + pwr_integral_gain: prop!("apple,pwr-integral-gain", f32!(0.0202129)), + pwr_integral_min_clamp: prop!("apple,pwr-integral-min-clamp", 0), + pwr_min_duty_cycle: prop!("apple,pwr-min-duty-cycle"), + pwr_proportional_gain: prop!("apple,pwr-proportional-gain", f32!(5.2831855)), + pwr_sample_period_aic_clks: prop!( + "apple,pwr-sample-period-aic-clks", + cfg.base_clock_hz / 1000 * power_sample_period + ), + se_engagement_criteria: prop!("apple,se-engagement-criteria", -1), + se_filter_time_constant: prop!("apple,se-filter-time-constant", 9), + se_filter_time_constant_1: prop!("apple,se-filter-time-constant-1", 3), + se_inactive_threshold: prop!("apple,se-inactive-threshold", 2500), + se_ki: prop!("apple,se-ki", f32!(-50.0)), + se_ki_1: prop!("apple,se-ki-1", f32!(-100.0)), + se_kp: prop!("apple,se-kp", f32!(-5.0)), + se_kp_1: prop!("apple,se-kp-1", f32!(-10.0)), + se_reset_criteria: prop!("apple,se-reset-criteria", 50), + se_target: if cfg.gpu_gen == GpuGen::G15 { prop!("apple,se-target", 1400) } else { 1400 }, + + perf_states, + power_zones, + csafr, + }) + } + + pub(crate) fn max_frequency_khz(&self) -> u32 { + self.perf_states[self.perf_max_pstate as usize].freq_hz / 1000 + } +} diff --git a/drivers/gpu/drm/asahi/m3/hw/agx3.rs b/drivers/gpu/drm/asahi/m3/hw/agx3.rs new file mode 100644 index 0000000000000..cb0c4fdbfa012 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/hw/agx3.rs @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +#[cfg(not(test))] +use crate::identity::{GpuGen, GpuVariant}; +#[cfg(test)] +#[path = "../identity.rs"] +mod identity; +#[cfg(test)] +use identity::{GpuGen, GpuVariant}; + +/// Identification-only configuration for one AGX3 SoC. +#[derive(Debug)] +pub(crate) struct SocConfig { + /// Chip ID in hex format (e.g. 0x6050 for t6050). + pub(crate) chip_id: u32, + /// Decoded design generation for this chip. + pub(crate) gpu_gen: GpuGen, + /// Decoded variant for this chip. + pub(crate) gpu_variant: GpuVariant, + /// Hardware family byte expected at `0xd04000[31:24]`. + pub(crate) hw_family: u8, + /// Hardware variant byte expected at `0xd04000[23:16]`. + pub(crate) hw_variant: u8, + /// Die count expected at `0xd04010[19:16]`. + pub(crate) num_dies: u32, + /// Input-address bits per UAT root (42 on every AGX3 part). + pub(crate) uat_ias: u8, + /// Output address space (42-bit on every AGX3 part). + pub(crate) uat_oas: u32, +} + +const fn agx3( + chip_id: u32, + gpu_gen: GpuGen, + gpu_variant: GpuVariant, + hw_family: u8, + hw_variant: u8, + num_dies: u32, +) -> SocConfig { + SocConfig { + chip_id, + gpu_gen, + gpu_variant, + hw_family, + hw_variant, + num_dies, + uat_ias: 42, + uat_oas: 42, + } +} + +pub(crate) const T8122: SocConfig = agx3(0x8122, GpuGen::G15, GpuVariant::G, 0x7, 2, 1); +/// Apple M3 Pro (G15S). +pub(crate) const T6030: SocConfig = agx3(0x6030, GpuGen::G15, GpuVariant::S, 0x7, 3, 1); +/// Apple M3 Max (G15C). +pub(crate) const T6031: SocConfig = agx3(0x6031, GpuGen::G15, GpuVariant::C, 0x7, 4, 1); +/// Apple M3 Ultra (G15D, two dies). +pub(crate) const T6032: SocConfig = agx3(0x6032, GpuGen::G15, GpuVariant::D, 0x7, 4, 2); diff --git a/drivers/gpu/drm/asahi/m3/hw/t8122.rs b/drivers/gpu/drm/asahi/m3/hw/t8122.rs new file mode 100644 index 0000000000000..5ba6b0ef6acd6 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/hw/t8122.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for T8122 (M3, G15G): one die, one cluster of ten core slots. +//! +//! Only the M3 runtime reads this table (`m3_soc::T8122`), and it refuses a T8122 until the +//! rest of the SoC's runtime configuration is known (`m3_soc::Soc::require_complete`). The +//! G15 manager backend never starts on T8122. + +use crate::f32; + +use super::*; + +/// Fender: ADT sgx reg[1] translated through the /arm-io ranges (SGX + 0xd00000). +const FENDER_BASE: usize = 0x2_90d0_0000; +/// The GPU register window: ADT sgx reg[0] translated through the /arm-io ranges. +const SGX_BASE: usize = 0x2_9000_0000; + +/// The firmware IO mappings, one per HwDataB slot (31 on G15). A slot is `None` when T8122 has +/// no block for it; its record then stays zero. Every address is a CPU physical address. +const IO_MAPPINGS: [Option; 31] = [ + // 0: Fender. The window is 0x104000 bytes: the ADT sgx reg[1] size (0x12c000) less 0x28000, + // the same difference as on T6030. + Some(IOMapping::new(FENDER_BASE, false, 1, 0x10_4000, 0, true)), + // 1: AIC timer, the same 16 KiB page on every SoC. + Some(IOMapping::new(0x2_0e10_0000, false, 1, 0x4000, 0, false)), + // 2: AIC software interrupts: AIC (0x2d1000000) + 0x14000, the page that holds the ADT's + // meta-sw-interrupt registers. + Some(IOMapping::new(0x2_d101_4000, false, 1, 0x4000, 0, true)), + // 3: RGX: the first 128 KiB of the GPU register window. + Some(IOMapping::new(SGX_BASE, false, 1, 0x2_0000, 0, true)), + None, // 4: UVD: the ADT sgx node has no third reg range + None, // 5: unused + None, // 6: display underrun workaround + None, // 7: analog temperature sensors + None, // 8: PMP doorbell + // 9: metrology sensors, SGX + 0xe08000. + Some(IOMapping::new(SGX_BASE + 0xe0_8000, false, 1, 0x8000, 0, true)), + // 10: GM GIFAF registers, a 4 KiB page at Fender + 0xd000. + Some(IOMapping::new(FENDER_BASE + 0xd000, false, 1, 0x1000, 0, true)), + // 11: memory cache, two instances of 0x58000 bytes (a multiple of 16 KiB), at 0x220000000 + // and 0x222000000. + Some(IOMapping::new(0x2_2000_0000, false, 2, 0x5_8000, 0x200_0000, true)), + None, // 12: AIC banked registers + None, // 13: PMGR scratch + None, // 14: NIA idle register, die 0 + None, // 15: NIA idle register, die 1 + None, // 16: CRE registers + None, // 17: streaming codec registers + // 18: telemetry dashboard, written by the firmware: the 4 KiB at +0x10000 of the PMP's + // telemetry window (0x2d03c0000, 0x14000 bytes). + Some(IOMapping::new(0x2_d03d_0000, false, 1, 0x1000, 0, true)), + // 19: telemetry dashboard, read by the firmware: the first 8 KiB of the same window. + Some(IOMapping::new(0x2_d03c_0000, false, 1, 0x2000, 0, false)), + None, // 20: telemetry dashboard configuration + None, // 21 + None, // 22 + None, // 23: AFR registers + None, // 24 + // 25: ANE doorbell, ANE + 0x45c000. + Some(IOMapping::new(0x3_1145_c000, false, 1, 0x4000, 0, true)), + // 26: PMS metrology sensors. + Some(IOMapping::new(0x2_d028_0000, false, 1, 0x8000, 0, false)), + None, // 27 + None, // 28 + // 29: GPU clock generator, SGX + 0xe1c000. + Some(IOMapping::new(SGX_BASE + 0xe1_c000, false, 1, 0x4000, 0, false)), + None, // 30 +]; + +/// T8122 (M3, G15G): one die, one cluster of ten core slots, eight or ten of them active. +pub(crate) const HWCONFIG_T8122: super::HwConfig = HwConfig { + chip_id: 0x8122, + gpu_gen: GpuGen::G15, // ID_VERSION[31:24] == 7 + gpu_variant: GpuVariant::G, // ID_VERSION[23:16] == 2 + gpu_core: Some(GpuCore::G15G), + + // The 24 MHz reference clock (HwDataB base_clock_khz = 24000). + base_clock_hz: 24_000_000, + // 42-bit UAT roots and output addresses, as on every AGX3 part (hw::agx3::T8122). + uat_ias: 42, + uat_oas: 42, + num_dies: 1, // ID_COUNTS_1[19:16] + max_num_clusters: 1, // one cluster of ten core slots + max_num_cores: 10, + max_num_frags: 10, // one fragment unit per core slot + max_num_gps: 4, // ID_COUNTS_2[23:16] + + // Not read by the M3 runtime, which keeps its own buffer layout. + preempt1_size: 0, + preempt2_size: 0, + preempt3_size: 0, + // Compute preemption buffer: 0xd80 bytes plus 0x700 per cluster. + compute_preempt1_size: 0x1480, + clustering: None, + render: HwRenderConfig { tiling_control: 0 }, + + da: HwConfigA { + // Cleared by the M3 runtime's InitData in any case. + unk_87c: 0, + // Fast-die temperature target, in centidegrees Celsius (110 C, as on T6030). The J613 + // ADT's gpu-fast-die0-target is 88; how the two relate is not known. + unk_8cc: 11_000, + unk_e24: 0, + }, + db: HwConfigB { + // The chip revision's major number: /arm-io chip-revision 0x20 >> 4. + unk_454: 2, + // The G15 InitData builders read none of the words below. + unk_4e0: 0, + unk_534: 0, + unk_ab8: 0, + unk_abc: 0, + unk_b30: 1, + }, + // G15 leaves the hws1/hws2/hws3 curves zero, as on T6030. + shared1_tab: &[], + shared1_a4: 0, + shared2_tab: &[], + shared2_unk_508: 0, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + // Idle-off standby timer when the device tree gives none: the J613 ADT has no + // gpu-idleoff-standby-timer. + idle_off_standby_timer_default: 700, + // The smart idle-off curve and count, the same as T6030's. + unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.7, 0.9])), + unk_hws2_24: 6, + // The M3 runtime writes 0xffff to this Globals word in any case. + global_unk_54: 0, + // SRAM power scale 1.02 for every performance state, as on T6030. + sram_k: f32!(1.02), + unk_coef_a: &[], + unk_coef_b: &[], + // No FwUtil performance tunables are written. + global_tab: None, + has_csafr: false, + // The fast-die sensor mask is each machine's: the M3 runtime takes it from the boot + // loader's apple,fast-die0-sensor-mask (the ADT's gpu-fast-die0-sensor-mask), not from here. + fast_sensor_mask: [0, 0], + fast_sensor_mask_alt: [0, 0], + fast_die0_sensor_present: 0, + io_mappings: &IO_MAPPINGS, + // No Fender scratch SRAM window. + sram_base: None, + sram_size: None, +}; diff --git a/drivers/gpu/drm/asahi/m3/identity.rs b/drivers/gpu/drm/asahi/m3/identity.rs new file mode 100644 index 0000000000000..f468272c5680e --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/identity.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +/// Supported GPU generation enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuGen { + G13 = 13, + G14 = 14, + G15 = 15, + G17 = 17, +} + +/// GPU variant enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuVariant { + P = 'P' as u32, + G = 'G' as u32, + S = 'S' as u32, + C = 'C' as u32, + D = 'D' as u32, +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuHalGeneration { + Legacy = 0, + Hal200 = 200, +} + +/// Public architecture axes for the G15 ID-register decoder. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub(crate) struct GpuIdentity { + pub(crate) gpu_gen: GpuGen, + pub(crate) gpu_variant: GpuVariant, + pub(crate) usc_generation: u32, + pub(crate) gpu_hal_generation: GpuHalGeneration, +} + +pub(crate) fn decode_gpu_identity(family: u8, variant: u8, num_dies: u8) -> Option { + use GpuVariant::*; + if family != 7 { return None; } + let (gpu_variant, usc_generation) = match variant { + 0 => (P, 2), + 2 => (G, 3), + 3 => (S, 3), + 4 => match num_dies { 1 => (C, 3), 2 => (D, 3), _ => return None }, + _ => return None, + }; + Some(GpuIdentity { gpu_gen: GpuGen::G15, gpu_variant, usc_generation, + gpu_hal_generation: GpuHalGeneration::Legacy }) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn g15_variants_keep_their_public_identity() { + assert_eq!(decode_gpu_identity(7, 3, 1).unwrap().gpu_variant, GpuVariant::S); + assert_eq!(decode_gpu_identity(7, 4, 1).unwrap().gpu_variant, GpuVariant::C); + assert_eq!(decode_gpu_identity(7, 4, 2).unwrap().gpu_variant, GpuVariant::D); + assert_eq!(decode_gpu_identity(7, 2, 1).unwrap().usc_generation, 3); + assert!(decode_gpu_identity(7, 4, 0).is_none()); + assert!(decode_gpu_identity(10, 2, 1).is_none()); + } +} diff --git a/drivers/gpu/drm/asahi/m3/initdata.rs b/drivers/gpu/drm/asahi/m3/initdata.rs new file mode 100644 index 0000000000000..dfeffd900eb12 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/initdata.rs @@ -0,0 +1,1457 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! GPU initialization data builder. +//! +//! The root of all interaction between the GPU firmware and the host driver is a complex set of +//! nested structures that we call InitData. This includes both GPU hardware/firmware configuration +//! and the pointers to the ring buffers and global data fields that are used for communication at +//! runtime. +//! +//! Many of these structures are poorly understood, so there are lots of hardcoded unknown values +//! derived from observing the InitData structures that macOS generates. + +use crate::f32; +use crate::fw::initdata::*; +use crate::fw::types::*; +use crate::module_parameters; +use crate::{ + driver::AsahiDevice, + gem, + gpu, + hw, + mmu, // +}; +use kernel::error::{ + Error, + Result, // +}; +use kernel::macros::versions; +use kernel::prelude::*; +use kernel::try_init; + +use ::pin_init; +use ::pin_init::Init; + +/// Builder helper for the global GPU InitData. + +/// G15 idle-off delay (ms, Globals +0x9a0/+0x9a4) used as "never idle off" while the rings live +/// in Fender scratch SRAM without a backup/restore path. +/// TODO: the firmware's encoding of "never" is unknown. i32::MAX stays positive if the +/// firmware treats the value as signed, and is never smaller than u32::MAX would be if it +/// converts ms to 24 MHz ticks in 32-bit arithmetic (both wrap to ~179 s); with 64-bit +/// arithmetic it is ~24 days. +#[allow(dead_code)] +pub(crate) const G15_KEEP_POWERED_MS: u32 = i32::MAX as u32; + +/// Which T6030 backend a G15 InitData is built for. The two differ where the manager backend +/// keeps the values it was validated with on J516S. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub(crate) enum G15Layout { + /// The G15 GpuManager backend: rings in Fender SRAM, the GPU never idles off. + Manager, + /// The serialized M3 runtime backend: rings in DRAM with finite idle-off, its own GPU and + /// firmware VA layout, and the values its InitData carries for the power, idle and + /// shader-engine controllers. + Runtime, +} + +/// G15 InitData choices that are not part of the hardware or device-tree configuration. +#[derive(Copy, Clone, Debug)] +pub(crate) struct G15Options { + /// The backend the InitData is built for. + pub(crate) layout: G15Layout, + /// Publish the performance states as two voltage-sorted tables (see [`PStateTables`]). + pub(crate) split_pstates: bool, + /// Highest performance state the firmware may use, as an index into the voltage-sorted + /// primary table (1 = lowest). None publishes the whole table. + pub(crate) cap: Option, + /// Firmware VA of the timestamp arena (HwDataB+0x28); None uses the manager's kernel range. + pub(crate) timestamp_base: Option, + /// Manager layout: write the PPM words and the Globals power targets as the runtime layout + /// does (`asahi.g15_debug` bit 55). The runtime layout always does. + pub(crate) reference_ppm: bool, + /// Runtime layout: the SoC's HwDataB configuration words and unit masks + /// (`m3_soc::Soc::hwdata_b`). None for the manager layout, which does not write them. + pub(crate) runtime_hwdata_b: Option<&'static G15RuntimeHwDataB>, + /// Accept operating points whose clusters have different voltages + /// (`m3_soc::Soc::per_cluster_voltages`); see [`PStateTables`]. + pub(crate) per_cluster_voltages: bool, +} + +/// The HwDataB words of the runtime backend's InitData that are neither its GPU VA layout nor +/// device-tree values: configuration words where the manager layout keeps 0 or its validated +/// values (including the A/B pair +0xa98/+0xb40 at 0), and the two unit masks. They are values +/// of one SoC's runtime InitData (`m3_soc::Soc::hwdata_b`), not derived from the device tree. +#[derive(Debug)] +pub(crate) struct G15RuntimeHwDataB { + pub(crate) unk_454: u32, + pub(crate) unk_464: u32, + pub(crate) unk_a7c: u64, + pub(crate) unk_a98: u32, + pub(crate) unk_abc: u64, + pub(crate) unk_ae4: u32, + pub(crate) unk_b20: u32, + pub(crate) unk_b24: u64, + /// +0xb40, the "UAT enabled" word of the manager layout. + pub(crate) unk_554: u32, + pub(crate) unk_17b8: u32, + /// Unit enable masks, +0x17c0/+0x17c8. + pub(crate) unit_mask_a: u64, + pub(crate) unit_mask_b: u32, + pub(crate) unk_1808: u32, + pub(crate) unk_1818: u32, +} + +impl G15Options { + /// The options of the G15 manager backend: its validated defaults, the performance-state + /// cap (`asahi.m3_max_pstate`) and the `asahi.g15_debug` A/B switches. + fn manager(cfg: &'static hw::HwConfig) -> G15Options { + let g15 = cfg.gpu_gen == hw::GpuGen::G15; + G15Options { + layout: G15Layout::Manager, + split_pstates: g15 + && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerSplitPstates), + cap: g15.then(|| { + crate::m3_params::max_pstate_param().unwrap_or(crate::m3_params::MAX_PSTATE_DEFAULT) + }), + timestamp_base: None, + reference_ppm: g15 + && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerReferencePpm), + runtime_hwdata_b: None, + per_cluster_voltages: false, + } + } + + /// The published performance-state tables for `pwr`. + pub(crate) fn tables(&self, pwr: &hw::PwrConfig) -> Result { + PStateTables::new(pwr, self.split_pstates, self.cap, self.per_cluster_voltages) + } + + /// RuntimePointers+0x2d0, a configuration word: 4 in the runtime backend's InitData, 0 (its + /// validated value) for the manager. + pub(crate) fn runtime_pointers_2d0(&self) -> u32 { + match self.layout { + G15Layout::Manager => 0, + G15Layout::Runtime => 4, + } + } +} + +/// Start of the USC (shader code) window in the runtime backend's GPU VA layout. +const RUNTIME_USC_BASE: u64 = 0x10_0000_0000; +/// The runtime backend's unknown-page address, the end of its 42-bit user VA range. +const RUNTIME_UNKNOWN_PAGE: u64 = 0x2ff_ffff_8000; + +/// Maximum number of entries of a firmware performance-state table, including the off state. +const PSTATE_TABLE_ENTRIES: usize = 16; + +/// The performance-state tables as published to the G15 firmware. +/// +/// The device tree lists the performance states in the bootloader's order: several states can +/// share one voltage, and the frequencies are not sorted. The firmware takes two tables sorted +/// by voltage: the primary one with the highest frequency of each voltage, the secondary one with +/// the lowest, plus the device-tree index of every entry. Entry 0 is the off state in both. +/// Every frequency and voltage comes from the device tree; nothing is interpolated. +/// +/// With per-cluster voltages (T6031, whose clusters have their own binned voltage tables), the +/// states are grouped by their whole voltage list instead: states that share their highest +/// cluster voltage must have the same voltage on every cluster, the groups are ordered by that +/// highest voltage, and every cluster's voltage must rise from one entry to the next. +#[derive(Copy, Clone, Debug)] +pub(crate) struct PStateTables { + /// Number of entries, including the off state. + pub(crate) len: usize, + /// Device-tree index of each primary-table entry. + pub(crate) primary: [u8; PSTATE_TABLE_ENTRIES], + /// Device-tree index of each secondary-table entry (split tables only). + pub(crate) secondary: Option<[u8; PSTATE_TABLE_ENTRIES]>, + /// Highest primary-table entry the firmware may select. + pub(crate) max: u32, +} + +impl PStateTables { + /// The primary/secondary split of `states`: one entry per distinct voltage, ascending. + fn split( + states: &[hw::PState], + ) -> Result<([u8; PSTATE_TABLE_ENTRIES], [u8; PSTATE_TABLE_ENTRIES], usize)> { + Self::split_for(states, false) + } + + /// [`Self::split`], accepting per-cluster voltages when `per_cluster` is set. + fn split_for( + states: &[hw::PState], + per_cluster: bool, + ) -> Result<([u8; PSTATE_TABLE_ENTRIES], [u8; PSTATE_TABLE_ENTRIES], usize)> { + let mut primary = [0u8; PSTATE_TABLE_ENTRIES]; + let mut secondary = [0u8; PSTATE_TABLE_ENTRIES]; + if states.len() < 2 || states.len() > u8::MAX as usize || states[0].freq_hz != 0 { + return Err(EINVAL); + } + let mut len = 1; + let mut last_mv = 0; + loop { + // The next voltage above `last_mv`, and its fastest and slowest states. + let mut next: Option<(u32, usize, usize)> = None; + for (i, ps) in states.iter().enumerate().skip(1) { + let mv = ps.max_volt_mv(); + // Every state after the off state runs, with one voltage for all clusters + // unless the SoC gives each cluster its own. + if ps.freq_hz == 0 || (!per_cluster && ps.volt_mv.iter().any(|v| *v != mv)) { + return Err(EINVAL); + } + if mv <= last_mv { + continue; + } + // States of one entry run every cluster at the same voltages. + if let (true, Some((best, hi, _))) = (per_cluster, next) { + if mv == best && states[hi].volt_mv[..] != ps.volt_mv[..] { + return Err(EINVAL); + } + } + next = match next { + Some((best, hi, lo)) if mv == best => Some(( + mv, + if ps.freq_hz > states[hi].freq_hz { i } else { hi }, + if ps.freq_hz < states[lo].freq_hz { i } else { lo }, + )), + Some((best, _, _)) if mv > best => next, + _ => Some((mv, i, i)), + }; + } + let Some((mv, hi, lo)) = next else { + break; + }; + if len >= PSTATE_TABLE_ENTRIES + || states[hi].freq_hz <= states[primary[len - 1] as usize].freq_hz + || states[lo].freq_hz <= states[secondary[len - 1] as usize].freq_hz + { + // Faster states must need a higher voltage in both tables. + return Err(EINVAL); + } + // With per-cluster voltages, every cluster's voltage rises with the entry. + if per_cluster && len > 1 { + let previous = &states[primary[len - 1] as usize].volt_mv; + let current = &states[hi].volt_mv; + if previous.len() != current.len() + || previous.iter().zip(current.iter()).any(|(p, c)| c <= p) + { + return Err(EINVAL); + } + } + primary[len] = hi as u8; + secondary[len] = lo as u8; + len += 1; + last_mv = mv; + } + // At least one running state must have a nonzero voltage; callers + // index the table up to len - 1 and clamp caps to 1..=len - 1. + if len < 2 { + return Err(EINVAL); + } + Ok((primary, secondary, len)) + } + + /// Build the published tables. `cap` is an index into the voltage-sorted primary table, + /// also when the states are published in device-tree order: the firmware may then use any + /// device-tree state up to the capped frequency, and the cap is translated to the highest + /// device-tree index whose states all stay at or below it. + /// `per_cluster` accepts per-cluster voltages (see [`PStateTables`]). + pub(crate) fn new( + pwr: &hw::PwrConfig, + split: bool, + cap: Option, + per_cluster: bool, + ) -> Result { + let states = &pwr.perf_states; + if split { + let (primary, secondary, len) = Self::split_for(states, per_cluster)?; + let top = len as u64 - 1; + let max = cap.map_or(top, |c| c.clamp(1, top)) as u32; + return Ok(PStateTables { + len, + primary, + secondary: Some(secondary), + max, + }); + } + if states.len() > PSTATE_TABLE_ENTRIES { + return Err(EINVAL); + } + let mut primary = [0u8; PSTATE_TABLE_ENTRIES]; + for (i, slot) in primary.iter_mut().enumerate().take(states.len()) { + *slot = i as u8; + } + let top = states.len() as u32 - 1; + let max = match cap { + // No voltage-sorted table can have more entries than the device tree has states, + // so such a cap leaves the whole table. + None => top, + Some(c) if c >= u64::from(top) => top, + Some(c) => { + let (sorted, _, sorted_len) = Self::split(states)?; + let limit_hz = states[sorted[c.clamp(1, sorted_len as u64 - 1) as usize] as usize].freq_hz; + // The longest device-tree prefix that stays at or below the capped frequency. + let mut max = 1; + while max < top && states[max as usize + 1].freq_hz <= limit_hz { + max += 1; + } + max + } + }; + Ok(PStateTables { + len: states.len(), + primary, + secondary: None, + max, + }) + } + + /// The device-tree state of primary-table entry `i`. + pub(crate) fn primary_state<'s>(&self, pwr: &'s hw::PwrConfig, i: usize) -> &'s hw::PState { + &pwr.perf_states[self.primary[i] as usize] + } + + /// The frequency of the highest entry the firmware may select, in kHz. + pub(crate) fn max_frequency_khz(&self, pwr: &hw::PwrConfig) -> u32 { + self.primary_state(pwr, self.max as usize).freq_hz / 1000 + } +} + +/// The highest GPU frequency the G15 manager backend lets the firmware use, in kHz (its +/// performance cap), or None when the GPU is not G15 or the cap cannot be applied. +pub(crate) fn g15_manager_max_frequency_khz( + cfg: &'static hw::HwConfig, + pwr: &hw::PwrConfig, +) -> Option { + if cfg.gpu_gen != hw::GpuGen::G15 { + return None; + } + let tables = G15Options::manager(cfg).tables(pwr).ok()?; + Some(tables.max_frequency_khz(pwr)) +} + +pub(crate) struct InitDataBuilderG15V14_8_3<'a> { + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: G15Options, +} + +impl<'a> InitDataBuilderG15V14_8_3<'a> { + /// Create a new InitData builder + pub(crate) fn new( + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> InitDataBuilderG15V14_8_3<'a> { + InitDataBuilderG15V14_8_3 { + dev, + alloc, + cfg, + dyncfg, + g15: G15Options::manager(cfg), + } + } + + /// Create an unknown T81xx-specific data structure. + fn t81xx_data( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> impl Init { + let _perf_max_pstate = dyncfg.pwr.perf_max_pstate; + + pin_init::init_zeroed::().chain(move |_ret| { + match cfg.chip_id { + 0x8103 | 0x8112 => { + } + _ => (), + } + Ok(()) + }) + } + + /// Build the HwDataA contents (mostly power-related configuration) without allocating the + /// object, so that they can fill a firmware object or a plain buffer alike. + pub(crate) fn hwdata_a_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result + 'a> { + let pwr = &dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms; + let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms; + let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into(); + let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into(); + let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into(); + let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into(); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms; + let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into(); + let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into(); + + let base_ps = pwr.perf_base_pstate; + let base_ps_scaled = 100 * base_ps; + let tables = g15.tables(pwr)?; + let max_ps = tables.max; + let runtime = g15.layout == G15Layout::Runtime; + let max_ps_scaled = 100 * max_ps; + let boost_ps_count = if cfg.gpu_gen == hw::GpuGen::G15 { + max_ps.checked_sub(base_ps).ok_or(EINVAL)? + } else { max_ps - base_ps }; + + #[allow(unused_variables)] + let base_clock_khz = cfg.base_clock_hz / 1000; + let v_clocks_per_period = pwr.pwr_sample_period_aic_clks; + + #[allow(unused_variables)] + let clocks_per_period_coarse = cfg.base_clock_hz / 1000 * pwr.power_sample_period; + + Ok( + try_init!(raw::HwDataAG15V14_8_3 { + clocks_per_period: v_clocks_per_period, + clocks_per_period_2: v_clocks_per_period, + pwr_status: AtomicU32::new(4), + unk_10: f32!(1.0), + actual_pstate: 1, + tgt_pstate: 1, + base_pstate_scaled: base_ps_scaled, + unk_40: 1, + max_pstate_scaled: max_ps_scaled, + min_pstate_scaled: 100, + unk_64c: 625, + pwr_filter_a_neg: f32!(1.0) - pwr_filter_a, + pwr_filter_a: pwr_filter_a, + pwr_integral_gain: pwr.pwr_integral_gain, + pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(), + max_power_1: pwr.max_power_mw.into(), + pwr_proportional_gain: pwr.pwr_proportional_gain, + pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(), + pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 - max_ps_scaled as i32, + max_pstate_scaled_2: max_ps_scaled, + max_power_2: pwr.max_power_mw, + max_pstate_scaled_3: max_ps_scaled, + ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4, + ppm_filter_a_neg: f32!(1.0) - ppm_filter_a, + ppm_filter_a: ppm_filter_a, + ppm_ki_dt: pwr.ppm_ki * period_s, + unk_6fc: f32!(65536.0), + ppm_kp: pwr.ppm_kp, + pwr_min_duty_cycle: pwr.pwr_min_duty_cycle, + max_pstate_scaled_4: max_ps_scaled, + unk_71c: f32!(0.0), + max_power_3: pwr.max_power_mw, + cur_power_mw_2: 0x0, + ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms, + ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz, + perf_tgt_utilization: pwr.perf_tgt_utilization, + perf_boost_min_util: pwr.perf_boost_min_util, + perf_boost_ce_step: pwr.perf_boost_ce_step, + perf_reset_iters: pwr.perf_reset_iters, + unk_774: 6, + unk_778: 1, + perf_filter_drop_threshold: pwr.perf_filter_drop_threshold, + perf_filter_a_neg: f32!(1.0) - perf_filter_a, + perf_filter_a2_neg: f32!(1.0) - perf_filter_a2, + perf_filter_a: perf_filter_a, + perf_filter_a2: perf_filter_a2, + perf_ki: pwr.perf_integral_gain, + perf_ki2: pwr.perf_integral_gain2, + perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(), + unk_79c: f32!(95.0), + perf_kp: pwr.perf_proportional_gain, + perf_kp2: pwr.perf_proportional_gain2, + boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95), + base_pstate_scaled_2: base_ps_scaled, + max_pstate_scaled_5: max_ps_scaled, + base_pstate_scaled_3: base_ps_scaled, + perf_tgt_utilization_2: pwr.perf_tgt_utilization, + base_pstate_scaled_4: base_ps_scaled, + unk_7fc: f32!(65536.0), + pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(), + max_pstate_scaled_6: max_ps_scaled.into(), + max_freq_mhz: pwr.max_freq_mhz, + pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle, + min_pstate_scaled_4: f32!(100.0), + max_pstate_scaled_7: max_ps_scaled, + unk_alpha_neg: f32!(0.8), + unk_alpha: f32!(0.2), + fast_die0_sensor_mask: U64(cfg.fast_sensor_mask[0]), + fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, + unk_87c: cfg.da.unk_87c, + unk_880: 0x4, + unk_894: f32!(1.0), + + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_8a8: f32!(65536.0), + fast_die0_kp: pwr.fast_die0_proportional_gain, + pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle, + max_pstate_scaled_8: max_ps_scaled, + max_pstate_scaled_9: max_ps_scaled, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + unk_8cc: cfg.da.unk_8cc, + max_pstate_scaled_10: max_ps_scaled, + max_pstate_scaled_11: max_ps_scaled, + unk_c2c: 1, + power_zone_count: pwr.power_zones.len() as u32, + max_power_4: pwr.max_power_mw, + max_power_5: pwr.max_power_mw, + max_power_6: pwr.max_power_mw, + avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a, + avg_power_target_filter_a: avg_power_target_filter_a, + avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_xperiod: period_ms * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_clks: period_ms + * pwr.avg_power_target_filter_tc + * base_clock_khz, + avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods, + avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a, + avg_power_filter_a: avg_power_filter_a, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + unk_d20: f32!(65536.0), + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + max_pstate_scaled_12: max_ps_scaled, + max_pstate_scaled_13: max_ps_scaled, + max_power_7: pwr.max_power_mw.into(), + max_power_8: pwr.max_power_mw, + avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms, + avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz, + max_pstate_scaled_14: max_ps_scaled, + t81xx_data <- Self::t81xx_data(cfg, dyncfg), + fast_die0_sensor_mask_2: U64(cfg.fast_sensor_mask[0]), + unk_e24: cfg.da.unk_e24, + unk_e28: 1, + fast_die0_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[0]), + unk_163c: 1, + unk_3644: 0, + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + for i in 0..tables.len { + raw.sram_k[i] = cfg.sram_k; + } + + // The only constants the G15 firmware expects in the HwDataA 0x10b4 block + // (+0x10f4 = 0x32, +0x11cc = {0, 65536.0}). + // unk_e10_0 sits at 0x10b4 (asserted in fw/initdata.rs), so +0x40 is 0x10f4 + // and +0x11c is 0x11d0. The runtime layout fills the whole block below. + { + raw.unk_e10_0.unk_40 = 0x32; + raw.unk_e10_0.unk_11c = f32!(65536.0); + } + + if let Some(csafr) = pwr.csafr.as_ref() { + for (i, coef) in csafr.leak_coef_afr.iter().enumerate() { + raw.aux_leak_coef.cs_1[i] = *coef; + raw.aux_leak_coef.cs_2[i] = *coef; + } + + for (i, coef) in csafr.leak_coef_cs.iter().enumerate() { + raw.aux_leak_coef.afr_1[i] = *coef; + raw.aux_leak_coef.afr_2[i] = *coef; + } + } + + for i in 0..dyncfg.id.num_clusters as usize { + if let Some(coef_a) = cfg.unk_coef_a.get(i) { + (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); + (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a); + } + if let Some(coef_b) = cfg.unk_coef_b.get(i) { + (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b); + (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b); + } + } + + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc; + raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc; + let filter_a = f32!(1.0) / pz.filter_tc.into(); + raw.power_zones[i].filter_a = filter_a; + raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a; + raw.power_zones[i].unk_10 = 1320000000; + } + + for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() { + raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 }; + } + + if runtime { + Self::hwdata_a_runtime(raw, pwr, max_ps_scaled, clocks_per_period_coarse); + } + + Ok(()) + })) + } + + /// Create the HwDataA structure. This mostly contains power-related configuration. + fn hwdata_a(&mut self) -> Result> { + let init = Self::hwdata_a_init(self.cfg, self.dyncfg, &self.g15)?; + let dev = self.dev; + let dyncfg = self.dyncfg; + self.alloc + .private + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { + init.chain(move |raw| { + if !dyncfg.hw_data_b.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::HwDataAG15V14_8_3 as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_data_a.len() { + matches = false; + dev_err!( + dev.as_ref(), + "!!! Hwdata A size mismatch: {} {}", + sla.len(), + dyncfg.hw_data_a.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { + if sla[i] != dyncfg.hw_data_a[i] { + matches = false; + dev_err!(dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(dev.as_ref(), "!!! Hwdata A match"); + } + } + } + Ok(()) + }) + }) + } + + /// The runtime backend's HwDataA words that the manager layout leaves at their defaults: the + /// shader-engine controller block at +0x10b4 in its G15 order (all inputs from the device + /// tree), and the leakage coefficient and flag words of the G15 tail. Words that the + /// runtime's InitData keeps at zero are cleared. + fn hwdata_a_runtime( + raw: &mut raw::HwDataAG15V14_8_3, + pwr: &hw::PwrConfig, + max_ps_scaled: u32, + clocks_per_period_coarse: u32, + ) { + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into(); + let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into(); + let se = &mut raw.unk_e10_0; + se.unk_40 = 50; + se.unk_44 = 1; + se.gpu_se_filter_a_neg = f32!(1.0) - filter_a; + se.gpu_se_filter_1_a_neg = f32!(1.0) - filter_1_a; + se.gpu_se_filter_a = filter_a; + se.gpu_se_filter_1_a = filter_1_a; + se.gpu_se_ki_dt = pwr.se_ki * period_s; + se.gpu_se_ki_1_dt = pwr.se_ki_1 * period_s; + se.unk_68 = f32!(65536.0); + se.gpu_se_kp = pwr.se_kp; + se.gpu_se_kp_1 = pwr.se_kp_1; + se.unk_78 = 100; + se.max_pstate_scaled_1 = max_ps_scaled; + se.min_pstate_scaled = 100; + se.unk_88 = f32!(8000.0); + // The shader-engine target (ADT gpu-se-tgt, 1400 on J514S and J516S; the device tree + // may override it). + se.se_target = pwr.se_target; + se.gpu_se_filter_time_constant_ms = pwr.se_filter_time_constant * period_ms; + se.gpu_se_filter_time_constant_1_ms = pwr.se_filter_time_constant_1 * period_ms; + se.gpu_se_filter_time_constant_clks = + U64((pwr.se_filter_time_constant * clocks_per_period_coarse).into()); + se.gpu_se_filter_time_constant_1_clks = + U64((pwr.se_filter_time_constant_1 * clocks_per_period_coarse).into()); + se.unk_b0 = f32!(65536.0); + se.unk_d8 = 8000; + se.unk_dc = 4; + se.gpu_se_inactive_threshold = pwr.se_inactive_threshold; + se.gpu_se_engagement_criteria = pwr.se_engagement_criteria; + se.unk_e8 = 2; + se.unk_ec = 4; + se.gpu_se_reset_criteria = pwr.se_reset_criteria; + se.unk_11c = f32!(65536.0); + se.unk_12c = 40; + se.max_pstate_scaled_2 = max_ps_scaled; + + // No fast-die release temperature, no +0x8b8 offset and no alternate sensor mask: the + // T6030 device tree has no source for them. + raw.fast_die0_release_temp_cc = 0; + raw.unk_87c = 0; + raw.fast_die0_sensor_mask_alt = U64(0); + raw.unk_3640 = f32!(5.0).to_bits(); + // The fused core leakage coefficient of this machine (device tree), first cluster table. + if let Some(coef) = pwr.core_leak_coef.first() { + raw.cluster_tables[0][0] = coef.to_bits(); + } + raw.unk_4298 = 1; + raw.unk_429c[0] = 1; + } + + // Compare a structure against the copy m1n1 left in the reserved-memory blob, if any + // (debug aid, same as the G13/G14 builders do inline). + fn compare_blob(dev: &AsahiDevice, what: &str, raw: &T, blob: &[u8]) { + if blob.is_empty() { + return; + } + // SAFETY: `raw` is a fully initialized firmware structure; viewing it as bytes is fine. + let sla = unsafe { + core::slice::from_raw_parts(raw as *const T as *const u8, core::mem::size_of::()) + }; + if sla.len() != blob.len() { + dev_err!( + dev.as_ref(), + "!!! {} size mismatch: {} {}", + what, + sla.len(), + blob.len() + ); + } + match sla.iter().zip(blob.iter()).position(|(a, b)| a != b) { + Some(i) => dev_err!(dev.as_ref(), "!!! {} first mismatch: {i}", what), + None => dev_info!(dev.as_ref(), "!!! {} match", what), + } + } + + // Build the G15 (14.8.3) HwDataB contents without allocating the object. Board values whose + // source is not known yet are left 0 and marked TODO. + // (Plain comments: the versions macro needs `fn` right after a gating #[ver].) + fn hwdata_b_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result + 'a> { + let pwr = &dyncfg.pwr; + let tables = g15.tables(pwr)?; + let runtime = g15.layout == G15Layout::Runtime; + let runtime_hwdata_b = g15.runtime_hwdata_b; + let timestamp_base = match g15.timestamp_base { + Some(base) => base, + None => mmu::kern_iova(cfg, gpu::IOVA_KERN_TIMESTAMP_RANGE.start), + }; + Ok(try_init!(raw::HwDataBG15V14_8_3 { + // VA-layout words, +0x0/+0x8 + unk_8: U64(0x6f_00000000), + unk_18: U64(0xffc00000), + // USC start: the G15 firmware takes 0x10_00000000 at +0x10/+0x18. Like the G13 + // builder, the manager leaves 0: drm/asahi passes the USC base per queue. + unk_20: U64(0), + unk_28: U64(0), + // "Unknown page": the G15 firmware layout uses 0x2ff_ffff8000 (+0x20) in its + // 42-bit user layout; the manager keeps a 39-bit user range and maps the page + // at IOVA_UNK_PAGE, so it passes that. + unk_30: U64(mmu::IOVA_UNK_PAGE), + // FW VA of the timestamp area, +0x28 + timestamp_area_base: U64(timestamp_base), + // TODO: CSC matrices (+0x38) and border-colour pointer (+0x638) + // are left 0, as on G13 ("TODO: yuv matrices"). + // +0xa28..+0xa34: chip id + 3 words (G13 unk_454..45c) + chip_id: cfg.chip_id, + unk_454: cfg.db.unk_454, + unk_458: 0x1, + // +0xa38..+0xa64 (G13 unk_460..unk_48c). +0xa48/+0xa54/+0xa64 are 0. + // +0xa38 (GPC present, 1 on t6030) and +0xa40 (PIO supported) are 1: the + // firmware runs its power management, and so powers up the GPU cores for + // a kicked job, only if both are non-zero. +0xa50/+0xa58/+0xa60 are 1. + // Tested on J516S: an empty compute job completes with these values. + unk_460: 1, + unk_468: 1, + unk_478: 1, + unk_480: 1, + unk_488: 1, + // +0xa68: 0x5dc0 = 24000 + base_clock_khz: cfg.base_clock_hz / 1000, + // +0xa6c: 200 on slow simulation models, otherwise 1, so 1 on t6030. The + // HwDataA filter constants follow +0xa70 (the sample period in ms), which + // stays the DT value. + power_sample_period: 1, + // +0xa70: the value HwDataA +0xea8/+0xeac multiply the filter time + // constants by, i.e. the sample period in ms (inferred) + unk_a70: pwr.power_sample_period, + // TODO: +0xa74, +0xa88, +0xa90, +0xaa0, +0xac4 are board/config values whose + // source is not known yet; left 0. + unk_a78: 1, + unk_a7c: U64(1), + unk_a98: 1, + unk_a9c: 0x1f, + unk_ac8: Array::new([0, 1, 1, 0]), + // TODO: +0xaa8 and +0xb04 are condition flags; left 0. + // "UAT enabled" flag, G13 unk_554 = 1. +0xb40 + unk_554: 0x1, + // +0xb44..+0xb58 + uat_ttb_base: U64(dyncfg.uat_ttb_base), + gpu_core_id: cfg.gpu_core.ok_or(ENODEV)? as u32, + gpu_rev_id: dyncfg.id.gpu_rev_id as u32, + num_cores: dyncfg.id.num_cores * dyncfg.id.num_clusters, + max_pstate: tables.len as u32 - 1, + // +0x17e8/+0x1800/+0x1804/+0x1810; + // +0x1814 = 1 as well. + unk_17e8: 1, + timer_offset: U64(0), + unk_1800: 1, + unk_1804: 1, + unk_1810: 1, + unk_1814: 1, + // +0x17fc = 1 on t6030. + unk_17fc: 1, + // +0x1820: 48 x 0xff + all_ones_masks: Array::new([0xff; 0x30]), + // +0x1860 = 1 & ~(debug global bit 0), so 1 without debug. (inferred) + unk_1860: 1, + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + // Rounded up, like the operating-point voltages. + let min_sram_mv = pwr.min_sram_microvolt.div_ceil(1000); + // Core and SRAM voltage rows of one state: every column holds the state's + // device-tree voltage (columns beyond the device tree's repeat the first one), and + // the SRAM rail never goes below the device tree's floor. + let rows = |ps: &hw::PState| -> ([u32; 8], [u32; 8]) { + let mut volts = [0u32; 8]; + for (j, v) in volts.iter_mut().enumerate() { + *v = *ps.volt_mv.get(j).unwrap_or(&ps.volt_mv[0]); + } + (volts, volts.map(|mv| mv.max(min_sram_mv))) + }; + + // p-state tables at +0xb58..+0xf9b, +0xfdc, same formulas as G13. Every + // frequency and voltage comes from the device tree. + for i in 0..tables.len { + let ps = tables.primary_state(pwr, i); + let (volts, sram) = rows(ps); + raw.frequencies[i] = ps.freq_hz / 1000000; + raw.voltages[i] = volts; + raw.voltages_sram[i] = sram; + raw.sram_k[i] = cfg.sram_k; + } + + match tables.secondary { + None => { + // One table in device-tree order: the second frequency table repeats it, + // and the index map is the identity. + for i in 0..tables.len { + raw.frequencies_2[i] = raw.frequencies[i]; + } + for i in 0..16 { + raw.unk_arr_0[i] = i as u32; + } + } + Some(secondary) => { + // The secondary table (+0xf9c), the device-tree index of every entry of + // both tables (+0x10dc, +0x111c) and the third table (+0x134c, the + // secondary table with one voltage column). + // The base state is a device-tree index, and the HwDataA base words use it as + // an index into the published table: the two must name the same state. + if pwr.perf_base_pstate as usize >= tables.len + || u32::from(tables.primary[pwr.perf_base_pstate as usize]) + != pwr.perf_base_pstate + { + return Err(EINVAL); + } + let top = tables.primary_state(pwr, tables.len - 1); + let base = tables.primary_state(pwr, pwr.perf_base_pstate as usize); + if top.pwr_mw == 0 || top.freq_hz <= base.freq_hz { + return Err(EINVAL); + } + let aux = &mut raw.aux_ps_3; + aux.max_pstate = tables.len as u32 - 1; + for i in 0..tables.len { + let ps = &pwr.perf_states[secondary[i] as usize]; + let (volts, sram) = rows(ps); + raw.frequencies_2[i] = ps.freq_hz / 1000000; + raw.unk_arr_0[i] = tables.primary[i] as u32; + raw.unk_arr_0[16 + i] = secondary[i] as u32; + aux.frequencies[i] = ps.freq_hz / 1000000; + aux.voltages[i][0] = volts[0]; + aux.voltages_sram[i][0] = sram[0]; + + // Maximum power and boost frequency of each primary entry, in percent + // of the top entry (boost relative to the base state). + let ps = tables.primary_state(pwr, i); + if i > 0 { + raw.rel_max_powers[i] = + (100 * u64::from(ps.pwr_mw) / u64::from(top.pwr_mw)) as u32; + } + if ps.freq_hz > base.freq_hz { + let f = u64::from(ps.freq_hz / 1000000 - base.freq_hz / 1000000); + let span = u64::from(top.freq_hz / 1000000 - base.freq_hz / 1000000); + raw.rel_boost_freqs[i] = (100 * f / span) as u32; + } + } + } + } + + if let Some(csafr) = pwr.csafr.as_ref() { + let aux = &mut raw.aux_ps; + aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?; + aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?; + + for (i, ps) in csafr.perf_states_cs.iter().enumerate() { + aux.cs_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.cs_voltages[i][j] = *mv; + aux.cs_voltages_sram[i][j] = sram_mv; + } + } + + for (i, ps) in csafr.perf_states_afr.iter().enumerate() { + aux.afr_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.afr_voltages[i][j] = *mv; + aux.afr_voltages_sram[i][j] = sram_mv; + } + } + } + + // +0x17c0 (u64) / +0x17c8 (u32): masks with the low N bits set + // (per-core / per-cluster enable masks, inferred). + // TODO: which counts these are; default total cores / clusters. + let low_bits = |n: u32| -> u64 { + if n >= 64 { + u64::MAX + } else { + (1u64 << n) - 1 + } + }; + raw.unit_mask_a = U64(low_bits(dyncfg.id.num_cores * dyncfg.id.num_clusters)); + raw.unit_mask_b = low_bits(dyncfg.id.num_clusters.min(32)) as u32; + + if runtime { + Self::hwdata_b_runtime(raw, runtime_hwdata_b.ok_or(EINVAL)?); + } + Ok(()) + })) + } + + // The runtime backend's HwDataB words: its GPU VA layout (USC base, unknown page), and the + // SoC's configuration words and unit masks (`words`, see `G15RuntimeHwDataB`). Those are a + // runtime InitData's values, not derived from the device tree: the T6030 ones hold for the + // boards that InitData was checked on (J514S, J516S), not necessarily for other + // configurations. + fn hwdata_b_runtime(raw: &mut raw::HwDataBG15V14_8_3, words: &G15RuntimeHwDataB) { + raw.unk_20 = U64(RUNTIME_USC_BASE); + raw.unk_28 = U64(RUNTIME_USC_BASE); + raw.unk_30 = U64(RUNTIME_UNKNOWN_PAGE); + raw.unk_454 = words.unk_454; + raw.unk_464 = words.unk_464; + raw.unk_a7c = U64(words.unk_a7c); + raw.unk_a98 = words.unk_a98; + raw.unk_abc = U64(words.unk_abc); + raw.unk_ae4 = words.unk_ae4; + raw.unk_b20 = words.unk_b20; + raw.unk_b24 = U64(words.unk_b24); + raw.unk_554 = words.unk_554; + raw.unk_17b8 = words.unk_17b8; + raw.unit_mask_a = U64(words.unit_mask_a); + raw.unit_mask_b = words.unit_mask_b; + raw.unk_1808 = words.unk_1808; + raw.unk_1818 = words.unk_1818; + } + + // Create the G15 (14.8.3) HwDataB structure. + fn hwdata_b(&mut self) -> Result> { + let init = Self::hwdata_b_init(self.cfg, self.dyncfg, &self.g15)?; + let dev = self.dev; + let dyncfg = self.dyncfg; + self.alloc + .private + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { + init.chain(move |raw| { + Self::compare_blob(dev, "Hwdata B", &*raw, &dyncfg.hw_data_b); + Ok(()) + }) + }) + } + + // Build the G15 (14.8.3) Globals contents without allocating the object. Only the fields + // with a known init value are set; the rest start at 0. + fn globals_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result + 'a> { + let pwr = &dyncfg.pwr; + let tables = g15.tables(pwr)?; + let runtime = g15.layout == G15Layout::Runtime; + let reference_ppm = g15.reference_ppm; + Ok(try_init!(raw::GlobalsG15V14_8_3 { + // TODO: 0 or 7 depending on a board condition (+0x0); 0 = timer + // context switch off. + timer_cswitch: 0, + // G13 unk_24 = 0, debug = 0 (inferred) + unk_28: 0, + debug: 0, + // +0x38: constant 0x78 + unk_38: 0x78, + // G13 unk_54/56/58 moved to +0x50..+0x54 (inferred) + // TODO: these are board configuration values; G13 values for now. + unk_50: cfg.global_unk_54, + unk_52: 40, + unk_54: 0xffff, + // G13 unk_5e = 1 moved to +0x5a (a flag on G15) (inferred) + unk_5a: U32(1), + // +0x5e = max(HwDataB+0xa6c, 1) = 1. + cswitch_timer_multiplier: U32(1), + // Must be non-zero before work is submitted. + // Non-zero at boot has been accepted by the firmware on J516S. + command_submission_enabled: 1, + pending_submissions: AtomicU32::new(0), + // G13 values. TODO: the G15 defaults for +0x97c..+0x984 are not + // known. + progress_check_interval_3d: 40, + progress_check_interval_ta: 10, + progress_check_interval_cl: 250, + // G13 unk_1102c_0.._c values; same group at +0x988 (inferred) + unk_1102c_0: 1, + unk_1102c_4: 1, + unk_1102c_8: 100, + unk_1102c_c: 1, + // +0x9a0..+0x9a8 + // TODO: with the rings in Fender SRAM the GPU and Fender must stay + // powered from firmware boot on, until an SRAM backup/restore exists. + // gpu.rs update_globals() keeps idle_off_delay_ms at the same value on + // every submit. + idle_off_delay_ms: AtomicU32::new(G15_KEEP_POWERED_MS), + fender_idle_off_delay_ms: G15_KEEP_POWERED_MS, + fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms, + // G13 values (+0x9b0/+0x9b4, set by setters) + cl_context_switch_timeout_ms: 40, + cl_kill_timeout_ms: 50, + // The manager leaves the FwUtil perf tunables (cfg.global_tab) and + // `soft_fault_settings` (+0xdf9) unwritten; the runtime layout sets them below. + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + if runtime { + Self::globals_runtime(raw, cfg, pwr, &tables)?; + } else if reference_ppm { + Self::globals_power_targets(raw, pwr, &tables); + } + Ok(()) + })) + } + + // The Globals power-interface targets (maximum power from the device tree, then the highest + // usable performance state scaled by 100, twice) and the performance-state cap. + fn globals_power_targets(raw: &mut raw::GlobalsG15V14_8_3, pwr: &hw::PwrConfig, tables: &PStateTables) { + let max_ps_scaled = 100 * tables.max; + raw.power_if_target = Array::new([pwr.max_power_mw, max_ps_scaled, max_ps_scaled]); + raw.perf_state_cap = max_ps_scaled; + } + + // The runtime backend's Globals: DRAM rings with finite idle-off, so the idle policy + // (imported by DeviceControl 0x13) and smart idle-off are on and the device-tree + // idle-off delays apply; the power interface targets and the performance-state cap + // (maximum power from the device tree, pstates scaled by 100); the smart idle-off + // configuration (standby timer from the device tree); and the FwUtil tunables. + fn globals_runtime( + raw: &mut raw::GlobalsG15V14_8_3, + cfg: &'static hw::HwConfig, + pwr: &hw::PwrConfig, + tables: &PStateTables, + ) -> Result { + raw.relaxed_cl_kill_timeout = 3000; + raw.debug = 1; + raw.smart_idle_off_enable = 1; + raw.unk_50 = 0xffff; + // The runtime enables command submission through its own queue protocol. + raw.command_submission_enabled = 0; + Self::globals_power_targets(raw, pwr, tables); + + let mut idle = [0u32; 10]; + idle[0] = pwr.idle_off_standby_timer; + if let Some(curve) = cfg.unk_hws2_4 { + for (slot, k) in idle[1..9].iter_mut().zip(curve.iter()) { + *slot = k.to_bits(); + } + } + idle[9] = cfg.unk_hws2_24; + raw.smart_idle_off_cfg = Array::new(idle); + raw.ut_engagement = 1; + raw.clvr_engagement = 1; + raw.keepalive_perf_threshold_rd = 100; + raw.keepalive_off_threshold_rd = 100; + + raw.idle_off_delay_ms = AtomicU32::new(pwr.idle_off_delay_ms); + raw.fender_idle_off_delay_ms = pwr.fender_idle_off_delay_ms; + raw.fw_early_wake_timeout_ms = pwr.fw_early_wake_timeout_ms; + + // CDM backoff 4, then a u32 1 and the FwUtil perf tunables from +0x9c1. + raw.cdm_backoff_timeout = 4; + raw.unk_9bd = Array::new([1, 0, 0, 0]); + if let Some(tab) = cfg.global_tab { + let mut bytes = [0u8; 0x1c]; + bytes.get_mut(..tab.len()).ok_or(EINVAL)?.copy_from_slice(tab); + raw.fwutil_default_fab_pstate = Array::new([bytes[0], bytes[1]]); + raw.fwutil_timer_period = bytes[2]; + (*raw.unk_9c4)[..0x19].copy_from_slice(&bytes[3..0x1c]); + } + raw.soft_fault_settings = U32(3); + Ok(()) + } + + // Create the G15 (14.8.3) Globals structure. + fn globals(&mut self) -> Result> { + let init = Self::globals_init(self.cfg, self.dyncfg, &self.g15)?; + let dev = self.dev; + let dyncfg = self.dyncfg; + self.alloc + .private + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { + init.chain(move |raw| { + Self::compare_blob(dev, "Globals", &*raw, &dyncfg.hw_globals); + Ok(()) + }) + }) + } + + // Create the G15 (14.8.3) RuntimePointers structure. The ring descriptors are filled in by + // the GPU manager. + fn runtime_pointers(&mut self) -> Result> { + let hwa = self.hwdata_a()?; + let hwb = self.hwdata_b()?; + let bufmgr_fw_addr = mmu::kern_iova(self.cfg, gpu::IOVA_KERN_GPU_BUFMGR_HIGH); + let rp_2d0 = self.g15.runtime_pointers_2d0(); + + let mut buffer_mgr_ctl = gem::new_kernel_object(self.dev, 0x4000)?; + buffer_mgr_ctl.vmap()?.memset(0); + + GpuObject::new_init_prealloc( + self.alloc.private.alloc_object()?, + |_ptr| { + let alloc = &mut *self.alloc; + try_init!(RuntimePointersG15V14_8_3 { + // G15 E1/E2/E3 are 0xc10/0x1248/0xe10 bytes; the G13 + // GpuGlobalStatsVtx/GpuStatsComp buffers are larger, which is harmless. + stats <- { + let alloc = &mut *alloc; + try_init!(StatsG15V14_8_3 { + vtx: alloc.private.new_default::()?, + frag: alloc.private.new_init( + pin_init::init_zeroed::(), + |_inner, _ptr| { + try_init!(raw::GpuGlobalStatsFragG15V14_8_3 { + total_cmds: 0, + unk_4: 0, + stats: Default::default(), + }) + } + )?, + comp: alloc.private.new_default::()?, + }) + }, + + hwdata_a: hwa, + hwdata_b: hwb, + + // BRN (erratum) table. + // TODO: size and contents unknown; one zeroed page (no errata). + brn_table: alloc.private.array_empty_tagged(0x4000, b"IBRN")?, + // E7: 0x88 bytes, zeroed + unkptr_e7: alloc.private.array_empty_tagged(0x88, b"IE7_")?, + // E5: 0x60 bytes, zeroed + unkptr_e5: alloc.private.array_empty_tagged(0x60, b"IE5_")?, + // UMA page-pool descriptor table (RuntimePointers +0x2c0/+0x2c8). + // TODO: size/format unknown; one zeroed page. The firmware takes a + // GPU VA and a FW VA; both point at this kernel mapping for now. + uma_pool_desc: alloc.shared.array_empty_tagged(0x4000, b"IUMA")?, + + buffer_mgr_ctl, + buffer_mgr_ctl_low_mapping: None, + buffer_mgr_ctl_high_mapping: None, + }) + }, + |inner, _ptr| { + try_init!(raw::RuntimePointersG15V14_8_3 { + hwdata_b: inner.hwdata_b.gpu_pointer(), + brn_table: inner.brn_table.gpu_pointer(), + unkptr_e7: inner.unkptr_e7.gpu_pointer(), + + pipes: Default::default(), + device_control: Default::default(), + event: Default::default(), + fw_log: Default::default(), + ktrace: Default::default(), + stats: Default::default(), + fwlog_buf: None, + __pad_200: Default::default(), + // FWLog enable: the firmware writes no log records while this is 0. + // Opt-in via asahi.g15_debug bit 43. + unk_230: crate::m3_params::g15_debug(crate::m3_params::G15Debug::FwLog) as u32, + + stats_vtx: inner.stats.vtx.gpu_pointer(), + stats_frag: inner.stats.frag.gpu_pointer(), + stats_comp: inner.stats.comp.gpu_pointer(), + unkptr_e5: inner.unkptr_e5.gpu_pointer(), + __pad_254: Default::default(), + // Optional pointer, left 0 + unkptr_2a8: U64(0), + buffer_mgr_ctl_gpu_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_LOW), + buffer_mgr_ctl_fw_addr: U64(bufmgr_fw_addr), + uma_pool_desc_gpu_addr: U64(inner.uma_pool_desc.gpu_va().get()), + uma_pool_desc_fw_addr: U64(inner.uma_pool_desc.gpu_va().get()), + // TODO: board configuration words; +0x2d4 left 0. + unk_2d0: rp_2d0, + unk_2d4: 0, + __pad_2d8: Default::default(), + // +0x3b0: 0xff, then 0x90 zero bytes + cswitch_hist_ctl: 0xff, + unk_3b1: Default::default(), + hwdata_a: inner.hwdata_a.gpu_pointer(), + // TODO: 16 board configuration bytes; left 0. + unk_449: Default::default(), + __pad_459: Default::default(), + unk_46d: Default::default(), + __pad_48d: Default::default(), + }) + }, + ) + } + + // Fill the G15 P0 status block. Host-initialised words are 0 except +0x45c4, which arms the + // firmware work-scheduler callbacks, and in the runtime layout the noise-suppression idle + // power-off state (+0x4), which the runtime's InitData starts at 1. + fn status_block_fill(raw: &mut raw::StatusBlock, g15: &G15Options) { + raw.flags.sched_callbacks = 1; + if g15.layout == G15Layout::Runtime { + raw.noise_suppression_idle_pwroff = 1; + } + } + + // Create the G15 P0 status block. + fn status_block(&mut self) -> Result> { + let g15 = self.g15; + let mut block = self.alloc.shared.new_default::()?; + block.with_mut(|raw, _inner| Self::status_block_fill(raw, &g15)); + Ok(block) + } + + // Build the G15 P1 power-controller block without allocating it. The fields keep the order + // and the formulas of the G13 13.5 Globals power fields (inferred). + fn power_ctl_block_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> impl Init + 'a { + let pwr = &dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let runtime = g15.layout == G15Layout::Runtime; + let reference_ppm = runtime || g15.reference_ppm; + + try_init!(raw::PowerCtlBlock { + power_zone_count: pwr.power_zones.len() as u32, + avg_power_filter_tc_periods: pwr.avg_power_filter_tc_ms / period_ms, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + avg_power_target_filter_tc: pwr.avg_power_target_filter_tc, + // G13 Globals unk_89bc..unk_89e0 order (inferred) + fast_die_temp_target: cfg.da.unk_8cc, + fast_die0_release_temp: 100 * pwr.fast_die0_release_temp, + unk_b0: cfg.da.unk_87c, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + fast_die0_kp: pwr.fast_die0_proportional_gain, + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + // +0xc0/+0xc4 are board values at init (also mirrored at HwDataA + // +0x934/+0x93c, G13 unk_8e8/unk_8f0, which the G13 path leaves 0). + // TODO: real values unknown; default 0 as in the G13 HwDataA mirror. + unk_c0: 0, + unk_c4: 0, + // +0xc8..+0xd8, manager layout: the G13 unk_89e0 = 1 flag (on G15 a + // condition flag), then the PPM target and the PPM kp. The PPM controller + // normally receives them at runtime; the G13 Globals values keep it from + // starting with a zero target. + ppm_words: Array::new([1, pwr.max_power_mw, pwr.ppm_kp.to_bits(), 0, 0]), + // +0xec: 1 at init + se_engagement: 1, + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc = pz.filter_tc; + } + if runtime { + // Runtime layout: no fast-die release temperature or +0xb0 offset (the + // T6030 device tree has no source for them). + raw.fast_die0_release_temp = 0; + raw.unk_b0 = 0; + } + if reference_ppm { + // 0 at +0xc8, then the enable flag, the PPM target and the PPM kp and ki*dt. + let ppm_ki_dt = pwr.ppm_ki * period_s; + raw.ppm_words = Array::new([ + 0, + 1, + pwr.max_power_mw, + pwr.ppm_kp.to_bits(), + ppm_ki_dt.to_bits(), + ]); + } + Ok(()) + }) + } + + // Create the G15 P1 power-controller block. + fn power_ctl_block(&mut self) -> Result> { + let init = Self::power_ctl_block_init(self.cfg, self.dyncfg, &self.g15); + self.alloc + .shared + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| init) + } + + // Create the G15 P2 debug block. ktrace stays disabled, as on G13. + fn debug_block(&mut self) -> Result> { + self.alloc.shared.new_default::() + } + + /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. + fn uat_level_info( + cfg: &'static hw::HwConfig, + index_shift: usize, + num_entries: usize, + ) -> raw::UatLevelInfo { + raw::UatLevelInfo { + index_shift: index_shift as _, + unk_1: 14, + unk_2: 14, + unk_3: 8, + unk_4: 0x4000, + num_entries: num_entries as _, + unk_8: U64(1), + unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), + index_mask: U64(((num_entries - 1) << index_shift) as u64), + } + } + + /// The three UatLevelInfo records of InitData: L1 ("page catalogue") entries are 8 with a + /// 39-bit IAS and 64 with the 42-bit G15 IAS (index mask 0x3f0_0000_0000). + pub(crate) fn uat_levels(cfg: &'static hw::HwConfig) -> [raw::UatLevelInfo; 3] { + let uat_l1_entries = 1usize << (cfg.uat_ias as usize - 36); + [ + Self::uat_level_info(cfg, 36, uat_l1_entries), + Self::uat_level_info(cfg, 25, 2048), + Self::uat_level_info(cfg, 14, 2048), + ] + } + + /// Build the top-level InitData object. + #[inline(never)] + pub(crate) fn build(&mut self) -> Result>> { + { + let pwr = &self.dyncfg.pwr; + let tables = self.g15.tables(pwr)?; + dev_info!( + self.dev.as_ref(), + "G15: performance cap {} -> {} state {} of 1..={} ({} MHz)\n", + self.g15.cap.unwrap_or(0), + if tables.secondary.is_some() { "voltage-sorted" } else { "device-tree" }, + tables.max, + tables.len - 1, + tables.max_frequency_khz(pwr) / 1000 + ); + } + let runtime_pointers = self.runtime_pointers()?; + let globals = self.globals()?; + // 14.x: FwStatus lives inside the P0 status block + let fw_status = self.status_block()?; + let power_ctl_block = self.power_ctl_block()?; + let debug_block = self.debug_block()?; + let shared_ro = &mut self.alloc.shared_ro; + + let obj = self.alloc.private.new_init( + try_init!(InitDataG15V14_8_3 { + unk_buf: shared_ro.array_empty_tagged(0x4000, b"IDTA")?, + runtime_pointers, + globals, + fw_status, + power_ctl_block, + debug_block, + }), + |inner, _ptr| { + let cfg = &self.cfg; + try_init!(raw::InitDataG15V14_8_3 { + ver_info: Array::new([0x0490, 0x8380, 0xe21e, 0x0c08]), + unk_buf: inner.unk_buf.gpu_pointer(), + unk_8: 0, + unk_c: 0, + runtime_pointers: inner.runtime_pointers.gpu_pointer(), + globals: inner.globals.gpu_pointer(), + // InitData +0x28/+0x2c: u32 0, u32 1 + unk_28: 0, + unk_2c: 1, + uat_page_size: 0x4000, + uat_page_bits: 14, + uat_num_levels: 3, + uat_level_info: Array::new(Self::uat_levels(cfg)), + // TODO: pmap descriptor +0x7c, meaning unknown; left 0. + unk_94: 0, + __pad_98: Default::default(), + debug_block: inner.debug_block.gpu_pointer(), + status_block: inner.fw_status.gpu_pointer(), + power_ctl_block: inner.power_ctl_block.gpu_pointer(), + }) + }, + )?; + Ok(KBox::new(obj, GFP_KERNEL)?) + } +} + +/// G15 InitData contents built into plain memory, for a backend that places the firmware objects +/// itself (the M3 runtime backend, whose object layout is fixed). +pub(crate) struct G15Contents { + pub(crate) hwdata_a: KVBox, + pub(crate) hwdata_b: KVBox, + pub(crate) globals: KVBox, + pub(crate) power_ctl_block: KVBox, + pub(crate) status_block: KVBox, + pub(crate) uat_levels: [raw::UatLevelInfo; 3], + /// The published performance-state tables. + pub(crate) tables: PStateTables, +} + +impl<'a> InitDataBuilderG15V14_8_3<'a> { + /// Build the G15 InitData contents with the same builders the manager backend uses, into + /// plain memory. Pointer fields stay zero; the caller relocates them. + pub(crate) fn g15_contents( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result { + let mut status_block = KVBox::init(pin_init::init_zeroed(), GFP_KERNEL)?; + Self::status_block_fill(&mut status_block, g15); + Ok(G15Contents { + hwdata_a: KVBox::init(Self::hwdata_a_init(cfg, dyncfg, g15)?, GFP_KERNEL)?, + hwdata_b: KVBox::init(Self::hwdata_b_init(cfg, dyncfg, g15)?, GFP_KERNEL)?, + globals: KVBox::init(Self::globals_init(cfg, dyncfg, g15)?, GFP_KERNEL)?, + power_ctl_block: KVBox::init(Self::power_ctl_block_init(cfg, dyncfg, g15), GFP_KERNEL)?, + status_block, + uat_levels: Self::uat_levels(cfg), + tables: g15.tables(&dyncfg.pwr)?, + }) + } +} + +/// The bytes of a firmware structure built by this module. +/// +/// The structures are plain `repr(C)` data, and every builder starts from a zeroed object, so +/// no byte is uninitialized. +pub(crate) fn raw_bytes(value: &T) -> &[u8] { + // SAFETY: `value` is a live, initialized `T` (see above); the slice covers exactly it and + // borrows it for its lifetime. + unsafe { core::slice::from_raw_parts((value as *const T).cast::(), size_of::()) } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs new file mode 100644 index 0000000000000..2fa384c910688 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs @@ -0,0 +1,1491 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +use core::mem::{offset_of, size_of}; + +use kernel::{ + c_str, device, + device::Core, + platform, + prelude::*, + time::{delay::fsleep, Delta}, +}; + +use crate::{ + float::F32, + fw::initdata::raw, + hw, initdata, + initdata::{G15Layout, G15Options}, + m3_config::Records, + m3_firmware::Firmware, + m3_init_layout as init, + m3_init_storage as storage, + m3_memory::Buffer, + m3_params::{self, G15Debug, InitDataParam, ThermalMode}, + m3_soc::{IoMapping, PstateTable, Soc}, + t8122_hwdata, + t8122_start::Experiment, +}; + +// The T8122 HwDataA +0x418c block (`t8122_hwdata`) covers exactly the G13-era leakage +// coefficient fields from core_leak_coef[4] to the end of aux_leak_coef, which the G15 builder +// leaves zero, and nothing the builder fills. +const _: () = { + type A = raw::HwDataAG15V14_8_3; + assert!(offset_of!(A, core_leak_coef) == 0x417c); + assert!(offset_of!(A, sram_leak_coef) == 0x419c); + assert!(offset_of!(A, aux_leak_coef) == 0x41bc); + assert!(t8122_hwdata::OFFSET == offset_of!(A, core_leak_coef) + 4 * size_of::()); + assert!( + t8122_hwdata::OFFSET + t8122_hwdata::SIZE + == offset_of!(A, aux_leak_coef) + size_of::() + ); + assert!(t8122_hwdata::OFFSET + t8122_hwdata::SIZE <= size_of::()); +}; + +/// Owner indices (`m3_init_storage`) of the objects the typed builders generate. +pub(crate) const INITDATA: usize = storage::ROOT; +pub(crate) const RUNTIME_POINTERS: usize = storage::RUNTIME_POINTERS; +pub(crate) const HWDATA: usize = storage::HARDWARE_DATA; +pub(crate) const GLOBALS: usize = storage::GLOBALS; +pub(crate) const GLOBALS_POWER: usize = storage::GLOBALS_POWER; +pub(crate) const STATUS: usize = storage::CONTROL_REGION; +pub(crate) const DEBUG: usize = storage::RUNTIME_FLAGS; +/// HwDataA follows HwDataB inside the HwData object. +pub(crate) const HWDATA_A: usize = 0x4580; + +/// Offset of HwDataB's IO-mapping table and the size of one entry. +const IO_MAPPINGS: usize = offset_of!(raw::HwDataBG15V14_8_3, io_mappings); +const IO_MAPPING_SIZE: usize = size_of::(); +/// Number of IO-mapping slots of HwDataB. +const IO_MAPPING_SLOTS: usize = 31; + +/// The T6030 firmware IO mappings of the runtime backend (HwDataB slot, physical address, total +/// size, element size, writable). The runtime maps IO with its own IO-mapping table +/// (`m3_init_storage::T6030_IOMAPS`), which must cover every entry (checked when the contents +/// are built). The physical addresses are T6030 SoC register blocks, the same ones the G15 manager +/// maps (`hw::t6030`). +pub(crate) const T6030_IO_MAPPINGS: [IoMapping; 15] = [ + (0, 0x2_90d0_0000, 0x14_4000, 0x14_4000, true), // Fender + (1, 0x2_0e10_1000, 1, 1, false), // AIC timer + (2, 0x3_5101_4000, 0x4000, 0x4000, true), // AIC software interrupts + (3, 0x2_9000_0000, 0x2_0000, 0x2_0000, true), // RGX + (7, 0x3_502b_c000, 0x1000, 0x1000, false), // analog temperature sensors + (9, 0x2_90e0_8000, 0x8000, 0x8000, true), // metrology sensors + (11, 0x2_2000_0000, 0x12_a000, 0x9_5000, true), // memory cache, two instances + (12, 0x3_5104_c000, 0x4000, 0x4000, false), // AIC banked registers + (18, 0x3_503d_0000, 0x4000, 0x4000, true), // telemetry dashboard + (19, 0x3_503c_0000, 0x4000, 0x4000, false), // telemetry dashboard (read) + (20, 0x3_503d_8000, 0x4000, 0x4000, true), // telemetry dashboard (config) + (23, 0x2_9300_0000, 0x40_0000, 0x40_0000, true), // AFR registers + (25, 0x3_0945_c000, 1, 1, true), // ANE doorbell + (26, 0x3_5028_0000, 0x8000, 0x8000, false), // PMS metrology sensors + (29, 0x2_90e5_c000, 0x4000, 0x4000, false), // GFX clock generator +]; + +/// The T8122 firmware IO mappings of the runtime backend, in the same form: the blocks of +/// `hw::t8122`, with the exact register, total size and element size the G15G firmware takes in +/// each slot. The AIC timer entry is the T6030 one, the same register on every SoC. +/// `fill_io_mappings` checks every entry against the runtime's IO maps of the SoC +/// ([`T8122_IOMAPS`]) before writing it. +pub(crate) const T8122_IO_MAPPINGS: [IoMapping; 12] = [ + (0, 0x2_90d0_0000, 0x10_4000, 0x10_4000, true), // Fender + (1, 0x2_0e10_1000, 1, 1, false), // AIC timer + (2, 0x2_d101_4048, 1, 1, true), // AIC software interrupt register + (3, 0x2_9000_0000, 0x2_0000, 0x2_0000, true), // RGX + (9, 0x2_90e0_8000, 0x8000, 0x8000, true), // metrology sensors + (10, 0x2_90d0_d000, 0x1000, 0x1000, true), // GM GIFAF registers + (11, 0x2_2000_0000, 0xa_a000, 0x5_5000, true), // memory cache, two instances + (18, 0x2_d03d_0000, 0x1000, 0x1000, true), // telemetry dashboard + (19, 0x2_d03c_0000, 0x2000, 0x2000, false), // telemetry dashboard (read) + (25, 0x3_1145_c000, 1, 1, true), // ANE doorbell + (26, 0x2_d028_0000, 0x8000, 0x8000, false), // PMS metrology sensors + (29, 0x2_90e1_c000, 0x4000, 0x4000, false), // GPU clock generator +]; + +/// The T8122 runtime's IO maps (`m3_soc::T8122.iomaps`): its IO mappings, laid out as the T6030 +/// ones are. +pub(crate) static T8122_IOMAPS: [storage::IoMap; 12] = + storage::pack_iomaps(&T8122_IO_MAPPINGS, storage::IOMAP_BASE); + +/// Whether every entry of `mappings` is a whole number of elements in the HwConfig block of the +/// same slot, with the same writability: the check `fill_io_mappings` makes when it builds the +/// InitData. +const fn same_blocks(mappings: &[IoMapping], cfg: &hw::HwConfig) -> bool { + let mut i = 0; + while i < mappings.len() { + let (slot, phys, total, element, writable) = mappings[i]; + if slot >= cfg.io_mappings.len() || element == 0 || total % element != 0 { + return false; + } + match cfg.io_mappings[slot] { + Some(block) + if block.base as u64 & !0x3fff == phys & !0x3fff && block.writable == writable => {} + _ => return false, + } + i += 1; + } + true +} + +const _: () = assert!(same_blocks(&T8122_IO_MAPPINGS, &hw::t8122::HWCONFIG_T8122)); + +// The T6030 IO maps are the packed IO maps of its IO mappings, the layout the T8122 start +// experiment derives its own from (`storage::pack_iomaps`). +const _: () = assert!(storage::same_iomaps( + &storage::pack_iomaps(&T6030_IO_MAPPINGS, storage::IOMAP_BASE), + &storage::T6030_IOMAPS +)); + +/// Highest performance state device-tree InitData may use without the driver's thermal limit +/// (`asahi.m3_thermal=off`), and the runtime cap the thermal limit starts at and falls back to +/// (the "safe cap"). The InitData's fast-die temperature controller words stay 0, so the +/// firmware does not limit the die temperature by itself. +pub(crate) const ADT_MAX_PSTATE_LIMIT: u32 = 5; + +/// The performance states the runtime backend lets the firmware use. +#[derive(Copy, Clone, Debug)] +pub(crate) struct PstatePolicy { + /// Highest entry of the published table. + pub(crate) table_max: u32, + /// Frequency of the highest entry of the published table, in MHz. + pub(crate) table_max_mhz: u32, + /// The ceiling: the highest entry the firmware is given (the InitData's highest-state + /// words). A reported state above it marks the GPU failed. + pub(crate) max: u32, + /// The runtime cap at boot (the Globals cap words): the highest entry the firmware may use + /// until the thermal limit changes it. Equal to `max` unless the thermal limit is on or + /// holding, which start here and never go below it. + pub(crate) safe: u32, + /// The entry requested at boot, at most `safe`. + pub(crate) boot: u32, + /// Frequency of the ceiling entry, in MHz. + pub(crate) max_mhz: u32, + /// Frequency of the `safe` entry, in MHz. + pub(crate) safe_mhz: u32, + /// Frequency of the boot entry, in MHz. + pub(crate) boot_mhz: u32, + /// Frequency of every entry of the published table, in MHz (entry 0 is the off state). + pub(crate) freqs: [u32; 16], + /// `asahi.m3_thermal`. + pub(crate) thermal: ThermalMode, +} + +impl PstatePolicy { + /// The highest frequency the GPU may run at this boot, for userspace: the ceiling's when + /// the thermal limit is on, the runtime cap's otherwise. + pub(crate) fn reported_max_mhz(&self) -> u32 { + match self.thermal { + ThermalMode::On => self.max_mhz, + ThermalMode::Off | ThermalMode::Hold => self.safe_mhz, + } + } + + /// Clamp the `asahi.m3_max_pstate` and `asahi.m3_boot_pstate` settings to the published table + /// of `hwdata` (the generated HwData object image), and to `ceiling` when given (the T8122 + /// start experiment's `asahi.t8122_pstate_cap`). + fn new(dev: &device::Device, hwdata: &[u8], ceiling: Option) -> Result { + let hwb_max = offset_of!(raw::HwDataBG15V14_8_3, max_pstate); + let hwb_freq = offset_of!(raw::HwDataBG15V14_8_3, frequencies); + let table_max = read_u32(hwdata, hwb_max)?; + if table_max == 0 || table_max >= 16 { + return Err(EINVAL); + } + let freq = |i: u32| read_u32(hwdata, hwb_freq + 4 * i as usize); + // Unset: the thermal limit is on. + let thermal = m3_params::thermal_param().unwrap_or(ThermalMode::On); + // Unset: the whole table when the thermal limit bounds the runtime cap, the default cap + // otherwise. + let requested = m3_params::max_pstate_param().unwrap_or(match thermal { + ThermalMode::Hold | ThermalMode::On => u64::from(table_max), + ThermalMode::Off => m3_params::MAX_PSTATE_DEFAULT, + }); + let mut max = requested.clamp(1, u64::from(table_max)) as u32; + if u64::from(max) != requested { + dev_info!( + dev, + "M3: asahi.m3_max_pstate={} clamped to {} (the table has states 1..={})\n", + requested, + max, + table_max + ); + } + if thermal == ThermalMode::Off && max > ADT_MAX_PSTATE_LIMIT { + dev_info!( + dev, + "M3: performance cap {} lowered to {}: device-tree InitData leaves the fast-die temperature controller off and the thermal limit is off (asahi.m3_thermal=off)\n", + max, + ADT_MAX_PSTATE_LIMIT + ); + max = ADT_MAX_PSTATE_LIMIT; + } + if let Some(ceiling) = ceiling { + if max > ceiling { + dev_info!( + dev, + "M3 G15G start: performance cap {} lowered to {} (asahi.t8122_pstate_cap)\n", + max, + ceiling + ); + max = ceiling; + } + } + // Without the thermal limit the runtime cap is the ceiling; with it, the runtime cap + // starts at (and never goes below) the cap used without it. + let safe = match thermal { + ThermalMode::Off => max, + ThermalMode::Hold | ThermalMode::On => max.min(ADT_MAX_PSTATE_LIMIT), + }; + let boot = m3_params::boot_pstate_param() + .unwrap_or(u64::from(safe)) + .clamp(1, u64::from(safe)) as u32; + let mut freqs = [0u32; 16]; + for (i, f) in freqs.iter_mut().enumerate().take(table_max as usize + 1) { + *f = freq(i as u32)?; + } + Ok(PstatePolicy { + table_max, + table_max_mhz: freqs[table_max as usize], + max, + safe, + boot, + max_mhz: freqs[max as usize], + safe_mhz: freqs[safe as usize], + boot_mhz: freqs[boot as usize], + freqs, + thermal, + }) + } +} + +/// The runtime backend's InitData contents: the generated object images and the +/// performance-state policy. +pub(crate) struct Contents { + /// Generated images, per owner; None for owners that stay zero. Only the three image + /// owners' images are uploaded; the records' are checked. + images: KVec>>, + /// The performance states the firmware may use. + pub(crate) pstates: PstatePolicy, + /// The runtime's IO maps of the SoC (`Soc::iomaps`), which cover the InitData IO mappings. + pub(crate) iomaps: &'static [storage::IoMap], + /// The accepted performance-state table (`Soc::pstates`): states above the off state, and + /// the frequency of the highest one in MHz. + pub(crate) table: (u32, u32), + /// The HwDataB slots whose IO maps the firmware gets read-only, as a bit mask. 0 (every IO + /// map read-write) except in the T8122 start experiment. + pub(crate) read_only_slots: u32, + /// The HwData object's placement and mapping (`Soc::hwdata_object`). + pub(crate) hwdata_object: Option, +} + +fn read_u32(bytes: &[u8], offset: usize) -> Result { + let word = bytes.get(offset..offset + 4).ok_or(EINVAL)?; + Ok(u32::from_le_bytes(word.try_into().map_err(|_| EINVAL)?)) +} + +fn write(bytes: &mut [u8], offset: usize, data: &[u8]) -> Result { + bytes + .get_mut(offset..offset + data.len()) + .ok_or(EINVAL)? + .copy_from_slice(data); + Ok(()) +} + +impl Contents { + /// Build the InitData contents from this board's device tree (`asahi.m3_initdata`), before + /// any GPU register is touched. + /// + /// `experiment` is the T8122 start experiment's values (`t8122_start`), on an armed T8122 + /// only; it then stands in for the SoC table's missing ones. `t6031` is the same for an + /// armed T6031 (`t6031_start`). + pub(crate) fn select( + pdev: &platform::Device, + firmware: &Firmware, + soc: &Soc, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, + ) -> Result { + let dev = pdev.as_ref(); + let param = m3_params::initdata_param(); + check_version(dev, firmware, experiment, t6031)?; + let iomaps = match experiment { + Some(e) => e.iomaps(), + None => match t6031 { + Some(e) => e.iomaps(), + None => soc.iomaps.ok_or(ENODEV)?, + }, + }; + storage::validate_iomaps(iomaps).map_err(|_| EINVAL)?; + let images = build_images(dev, firmware, soc, experiment, t6031, iomaps)?; + let hwdata = images.get(HWDATA).and_then(|i| i.as_deref()).ok_or(EINVAL)?; + let pstates = PstatePolicy::new( + dev, + hwdata, + experiment.map(|e| e.pstate_cap()).or_else(|| t6031.map(|e| e.pstate_cap())), + )?; + let table = match soc.pstates { + PstateTable::Fixed { states, top_mhz } => (states, top_mhz), + PstateTable::DeviceTree => crate::m3_board::opp_table_shape(pdev, soc).ok_or_else(|| { + dev_err!( + dev, + "M3: the device tree's operating points give no performance-state table\n" + ); + EINVAL + })?, + }; + if (pstates.table_max, pstates.table_max_mhz) != table { + dev_err!( + dev, + "M3: performance-state table 1..={} up to {} MHz, the {} table is 1..={} up to {} MHz\n", + pstates.table_max, + pstates.table_max_mhz, + soc.name, + table.0, + table.1 + ); + return Err(EINVAL); + } + // The published states never go above the device tree's highest operating point. + if let Some(dt_khz) = crate::m3_board::max_frequency_khz(pdev) { + if 1000 * pstates.max_mhz > dt_khz { + dev_err!( + dev, + "M3: performance cap {} MHz is above the device tree's highest operating point ({} kHz)\n", + pstates.max_mhz, + dt_khz + ); + return Err(EINVAL); + } + } + dev_info!( + dev, + "M3: InitData contents: generated from the device tree (asahi.m3_initdata={}); performance states 1..={}, cap {} ({} MHz), boot request {} ({} MHz); thermal limit {}, firmware highest state {} ({} MHz)\n", + match param { + InitDataParam::Auto => "auto", + InitDataParam::Adt => "adt", + }, + pstates.table_max, + pstates.safe, + pstates.safe_mhz, + pstates.boot, + pstates.boot_mhz, + match pstates.thermal { + ThermalMode::Off => "off", + ThermalMode::Hold => "holding", + ThermalMode::On => "on", + }, + pstates.max, + pstates.max_mhz + ); + let read_only_slots = experiment + .map(|e| e.read_only_slots()) + .or_else(|| t6031.map(|e| e.read_only_slots())) + .unwrap_or(0); + let hwdata_object = match t6031 { + Some(e) => e.hwdata_object(), + None => soc.hwdata_object, + }; + Ok(Contents { images, pstates, iomaps, table, read_only_slots, hwdata_object }) + } + + /// Check the contents before they are uploaded. + pub(crate) fn check_for_upload(&self) -> Result { + if self.images.len() == storage::COUNT { + Ok(()) + } else { + Err(EINVAL) + } + } + + /// The initial contents of owner `index`, allocated as `a`: its generated image, or nothing + /// (zeroed) for owners without one. + pub(crate) fn initial<'a>(&'a self, index: usize, a: &storage::Allocation) -> Result<&'a [u8]> { + if a.initial == storage::Initial::Zero { + return Ok(&[]); + } + let image = self.images.get(index).and_then(|i| i.as_deref()).ok_or(EINVAL)?; + if image.len() != a.size { + return Err(EINVAL); + } + Ok(image) + } +} + +/// The fixed allocation of owner `index`, as a firmware region moved by `delta`. +fn allocation_region(index: usize, delta: u64) -> Result { + let a = storage::allocation(index).map_err(|_| EINVAL)?; + init::Region::new(a.address.checked_add(delta).ok_or(EINVAL)?, a.size).map_err(|_| EINVAL) +} + +/// The records the runtime constructs, encoded for its fixed allocation table moved by `delta`. +pub(crate) fn allocation_records(delta: u64) -> Result { + Records::encode(|index| allocation_region(index, delta)) +} + +/// The version word of the constructed InitData root. +pub(crate) fn constructed_version() -> Result { + let records = allocation_records(0)?; + Ok(u64::from_le_bytes(records.root[..8].try_into().map_err(|_| EINVAL)?)) +} + +/// Refuse to go on when the InitData root the runtime constructs carries another version than +/// the loaded firmware expects. The T8122 start experiment gives the firmware the version of +/// `asahi.t8122_initdata_version` instead (`m3_config::Config::new` writes it into the root). +fn check_version( + dev: &device::Device, + firmware: &Firmware, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, +) -> Result { + let version = constructed_version()?; + if version != firmware.initdata_magic && experiment.is_some() { + dev_warn!( + dev, + "M3 G15G start: InitData version {:#x} (asahi.t8122_initdata_version) replaces the constructed {:#x}\n", + firmware.initdata_magic, + version + ); + return Ok(()); + } + if version != firmware.initdata_magic && t6031.is_some() { + dev_warn!( + dev, + "M3 G15C start: InitData version {:#x} replaces the constructed {:#x}\n", + firmware.initdata_magic, + version + ); + return Ok(()); + } + if version != firmware.initdata_magic { + dev_err!( + dev, + "M3: the constructed InitData version {:#x} is not the one the loaded firmware expects ({:#x})\n", + version, + firmware.initdata_magic + ); + return Err(ENODEV); + } + Ok(()) +} + +/// Whether the runtime probe stops before the GPU coprocessor starts (`asahi.g15_debug` bit +/// 53). Logs the stop. +pub(crate) fn stop_before_asc(dev: &device::Device) -> bool { + let stop = m3_params::g15_debug(G15Debug::StopBeforeAsc); + if stop { + dev_info!( + dev, + "M3: asahi.g15_debug bit 53: stopping before the GPU coprocessor starts; the firmware was not started\n" + ); + } + stop +} + +/// The GPU identity the runtime backend admits for `soc` (its SGX ID words, checked in +/// `m3_device` before the firmware starts): on T6030, G15, variant S, revision B1, one die with +/// two clusters of ten core slots. +fn soc_identity( + soc: &Soc, + cfg: &'static hw::HwConfig, + rev_id: Option, +) -> hw::GpuIdConfig { + hw::GpuIdConfig { + gpu_gen: hw::GpuGen::G15, + gpu_variant: soc.gpu_variant, + usc_generation: 3, + gpu_hal_generation: hw::GpuHalGeneration::Legacy, + gpu_rev: soc.gpu_revision, + gpu_rev_id: rev_id.unwrap_or(soc.gpu_revision_id), + num_dies: cfg.num_dies, + num_clusters: cfg.max_num_clusters, + num_cores: cfg.max_num_cores, + num_frags: cfg.max_num_frags, + num_gps: cfg.max_num_gps, + total_active_cores: 0, + core_masks: KVec::new(), + core_masks_packed: KVec::new(), + } +} + +pub(crate) const BOARD_DATA: [usize; 3] = [HWDATA, GLOBALS, GLOBALS_POWER]; + +/// The pointers of the constructed records that the typed firmware structures place: record +/// owner, offset, target owner, offset into the target. +const RECORD_POINTERS: [(usize, usize, usize, usize); 7] = { + type InitData = raw::InitDataG15V14_8_3<'static>; + type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>; + [ + (INITDATA, offset_of!(InitData, runtime_pointers), RUNTIME_POINTERS, 0), + (INITDATA, offset_of!(InitData, globals), GLOBALS, 0), + (INITDATA, offset_of!(InitData, debug_block), DEBUG, 0), + (INITDATA, offset_of!(InitData, status_block), STATUS, 0), + (INITDATA, offset_of!(InitData, power_ctl_block), GLOBALS_POWER, 0), + (RUNTIME_POINTERS, offset_of!(RuntimePointers, hwdata_b), HWDATA, 0), + (RUNTIME_POINTERS, offset_of!(RuntimePointers, hwdata_a), HWDATA, HWDATA_A), + ] +}; + +/// The bytes of a constructed record, by owner. +pub(crate) fn record_bytes(records: &Records, owner: usize) -> Option<&[u8]> { + match owner { + INITDATA => Some(&records.root[..]), + RUNTIME_POINTERS => Some(&records.runtime[..]), + STATUS => Some(&records.control[..]), + _ => None, + } +} + +fn check_layout() -> Result { + type InitData = raw::InitDataG15V14_8_3<'static>; + type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>; + let records = allocation_records(0)?; + let pointer = |owner: usize, offset: usize| -> Option { + let bytes = record_bytes(&records, owner)?.get(offset..offset + 8)?; + Some(u64::from_le_bytes(bytes.try_into().ok()?)) + }; + let mut ok = true; + for (owner, offset, target, at) in RECORD_POINTERS { + let a = storage::allocation(target).map_err(|_| EINVAL)?; + ok &= at < a.size && pointer(owner, offset) == Some(a.address + at as u64); + } + for index in 0..storage::COUNT { + let a = storage::allocation(index).map_err(|_| EINVAL)?; + let board = BOARD_DATA.contains(&index); + ok &= board == (a.initial != storage::Initial::Zero); + } + let size = |index: usize| storage::allocation(index).map_or(0, |a| a.size); + let ok = ok + && size(INITDATA) >= size_of::() + && size(RUNTIME_POINTERS) >= offset_of!(RuntimePointers, hwdata_a) + 8 + && HWDATA_A >= size_of::() + && size(HWDATA) >= HWDATA_A + size_of::() + && size(GLOBALS) >= size_of::() + && size(GLOBALS_POWER) >= size_of::() + && size(STATUS) >= size_of::() + && size(DEBUG) >= size_of::(); + if ok { Ok(()) } else { Err(EINVAL) } +} + +/// Fill HwDataB's IO-mapping table (the runtime fills the virtual addresses when it maps them), +/// checking every entry against the runtime's IO mapping of the same slot (`iomaps`). +fn fill_io_mappings( + dev: &device::Device, + cfg: &'static hw::HwConfig, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, + mappings: &[IoMapping], + iomaps: &[storage::IoMap], + hwdata: &mut [u8], +) -> Result { + for (i, &(slot, phys, total, element, writable)) in mappings.iter().enumerate() { + if slot >= 31 || mappings[..i].iter().any(|&(other, ..)| other == slot) { + return Err(EINVAL); + } + let entry = IO_MAPPINGS + slot * IO_MAPPING_SIZE; + let virt = entry + offset_of!(raw::IOMapping, virt_addr); + // The same register block as the manager's table. + let same_block = cfg.io_mappings.get(slot).and_then(|m| m.as_ref()).is_some_and(|m| { + let base = match (experiment, t6031) { + (Some(e), _) => e.io_block(slot, m.base as u64), + (None, Some(e)) => e.io_block(slot, m.base as u64), + (None, None) => m.base as u64, + }; + base & !0x3fff == phys & !0x3fff && m.writable == writable + }); + let covered = iomaps.iter().any(|io| { + io.slot == slot + && io.pointer_field() == Ok(virt) + && io.covers(phys, total) + }); + if !same_block || !covered || element == 0 || total % element != 0 { + dev_err!(dev, "M3: IO mapping slot {} ({:#x}) does not match the object layout\n", slot, phys); + return Err(EINVAL); + } + write(hwdata, entry + offset_of!(raw::IOMapping, phys_addr), &phys.to_le_bytes())?; + write(hwdata, entry + offset_of!(raw::IOMapping, total_size), &total.to_le_bytes())?; + write(hwdata, entry + offset_of!(raw::IOMapping, element_size), &element.to_le_bytes())?; + write( + hwdata, + entry + offset_of!(raw::IOMapping, readwrite), + &u64::from(writable).to_le_bytes(), + )?; + } + // Every IO mapping the runtime makes must belong to one of the entries above. + for io in iomaps.iter() { + if !mappings.iter().any(|&(slot, ..)| slot == io.slot) { + dev_err!(dev, "M3: IO mapping slot {} ({:#x}) has no device-tree InitData entry\n", io.slot, io.physical); + return Err(EINVAL); + } + } + Ok(()) +} + +pub(crate) fn check_upload( + dev: &device::Device, + firmware: &Firmware, + iomaps: &[storage::IoMap], + objects: &mut [Buffer], +) -> Result { + let mut bad = 0u32; + let version = objects.get_mut(INITDATA).ok_or(EINVAL)?.read_u64(0)?; + if version != firmware.initdata_magic { + dev_err!( + dev, + "M3: InitData self-check: version {:#x}, the loaded firmware expects {:#x}\n", + version, + firmware.initdata_magic + ); + bad += 1; + } + for (owner, offset, target, at) in RECORD_POINTERS { + let expected = objects.get(target).ok_or(EINVAL)?.va() + at as u64; + let value = objects.get_mut(owner).ok_or(EINVAL)?.read_u64(offset)?; + if value != expected { + dev_err!( + dev, + "M3: InitData self-check: owner {} +{:#x} holds {:#x}, not owner {} +{:#x} ({:#x})\n", + owner, + offset, + value, + target, + at, + expected + ); + bad += 1; + } + } + let hwdata = objects.get_mut(HWDATA).ok_or(EINVAL)?; + for slot in 0..IO_MAPPING_SLOTS { + let entry = IO_MAPPINGS + slot * IO_MAPPING_SIZE; + let phys = hwdata.read_u64(entry + offset_of!(raw::IOMapping, phys_addr))?; + let virt = hwdata.read_u64(entry + offset_of!(raw::IOMapping, virt_addr))?; + let total = u64::from(hwdata.read_u32(entry + offset_of!(raw::IOMapping, total_size))?); + let ok = match iomaps.iter().find(|io| io.slot == slot) { + Some(io) => { + phys == io.physical + io.offset as u64 + && virt == io.address + io.offset as u64 + && total != 0 + && phys + total <= io.physical + io.size as u64 + } + None => phys == 0 && virt == 0, + }; + if !ok { + dev_err!( + dev, + "M3: InitData self-check: IO mapping slot {} describes {:#x}+{:#x} at {:#x}, which the runtime does not map there\n", + slot, + phys, + total, + virt + ); + bad += 1; + } + } + let region = hwdata.read_u64(storage::GPU_REGION_PHYSICAL)?; + if region != firmware.resources.regions[0].base { + dev_err!( + dev, + "M3: InitData self-check: GPU region {:#x}, the resources admitted {:#x}\n", + region, + firmware.resources.regions[0].base + ); + bad += 1; + } + if bad != 0 { + dev_err!(dev, "M3: InitData self-check failed ({} findings); not publishing the InitData\n", bad); + return Err(EINVAL); + } + dev_info!( + dev, + "M3: InitData self-check passed: version {:#x}, {} record pointers, {} IO mappings, GPU region {:#x}\n", + version, + RECORD_POINTERS.len(), + iomaps.len(), + region + ); + Ok(()) +} + +/// The whole part of a positive `F32`, for logs. +fn whole(v: F32) -> u32 { + let bits = v.to_bits(); + match ((bits >> 23) & 0xff) as i32 - 127 { + e if e < 0 => 0, + e if e > 31 => u32::MAX, + e if e > 23 => ((bits & 0x7f_ffff) | 0x80_0000) << (e - 23), + e => ((bits & 0x7f_ffff) | 0x80_0000) >> (23 - e), + } +} + +/// The boot loader's decoded GPU leakage fuse values (`Soc::leak_fuse`), logged when it exported +/// them. Returns them when the boot loader's switch is set. With the switch set and no two +/// valid values, refuses: the boot loader then used neither the fuse nor the stand-in. +fn leak_fuse(dev: &device::Device, lf: &crate::m3_soc::LeakFuse) -> Result> { + let chosen = kernel::of::chosen(); + let switch = chosen + .as_ref() + .and_then(|c| c.get_property::>(lf.switch).ok()) + .is_some_and(|v| v.as_slice() == b"1\0"); + let values: Option> = chosen.as_ref().and_then(|c| c.get_property(lf.values).ok()); + // Positive, finite, nonzero. + let valid = |v: &F32| { + let b = v.to_bits(); + b != 0 && b >> 31 == 0 && (b >> 23) & 0xff != 0xff + }; + let pair = values + .as_ref() + .filter(|v| v.len() == 2 && v.iter().all(valid)) + .map(|v| [v[0], v[1]]); + let names = (lf.values.to_str().unwrap_or("?"), lf.switch.to_str().unwrap_or("?")); + match (pair, switch) { + (Some([a, b]), true) => { + dev_info!( + dev, + "M3: GPU leakage from the boot loader's fuse values: {} and {} (/chosen/{}, {} set)\n", + whole(a), + whole(b), + names.0, + names.1 + ); + Ok(Some([a, b])) + } + (Some([a, b]), false) => { + dev_info!( + dev, + "M3: the boot loader's GPU leakage fuse values are {} and {} (/chosen/{}); not used: {} is not set\n", + whole(a), + whole(b), + names.0, + names.1 + ); + Ok(None) + } + (None, true) => { + dev_err!( + dev, + "M3: {} is set, but /chosen/{} does not hold two valid values; GPU startup disabled\n", + names.1, + names.0 + ); + Err(EINVAL) + } + (None, false) => { + if values.is_some() { + dev_warn!(dev, "M3: /chosen/{} is malformed; ignored\n", names.0); + } + Ok(None) + } + } +} + +/// A zeroed image of owner `index`. +fn zeroed(index: usize) -> Result> { + let size = storage::allocation(index).map_err(|_| EINVAL)?.size; + Ok(KVVec::from_elem(0u8, size, GFP_KERNEL)?) +} + +/// Build the images of the generated owners from the device tree. +fn build_images( + dev: &device::Device, + firmware: &Firmware, + soc: &Soc, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, + iomaps: &[storage::IoMap], +) -> Result>>> { + check_layout().inspect_err(|_| { + dev_err!(dev, "M3: the constructed InitData records do not match the G15 InitData structures\n") + })?; + let cfg: &'static hw::HwConfig = match t6031 { + Some(e) => e.hwcfg(), + None => soc.hwcfg.ok_or(ENODEV)?, + }; + let io_mappings = match experiment { + Some(e) => e.io_mappings(), + None => match t6031 { + Some(e) => e.io_mappings(), + None => soc.io_mappings.ok_or(ENODEV)?, + }, + }; + let runtime_hwdata_b = match experiment { + Some(e) => e.hwdata_b(), + None => match t6031 { + Some(e) => e.hwdata_b(), + None => soc.hwdata_b.ok_or(ENODEV)?, + }, + }; + let mut pwr = hw::PwrConfig::load(dev, cfg).inspect_err(|e| { + dev_err!(dev, "M3: cannot read the GPU power configuration from the device tree ({:?})\n", e) + })?; + // The boot loader's decoded leakage fuse, when its switch makes it this boot's leakage: value + // 1 is the core leakage coefficient (the boot loader then writes it as apple,core-leak-coef + // too), value 2 goes to HwDataA's third leakage table below. + let fuse = match soc.leak_fuse { + Some(lf) => leak_fuse(dev, &lf)?, + None => None, + }; + if let Some([core, _]) = fuse { + match pwr.core_leak_coef.first_mut() { + Some(c) if c.to_bits() != core.to_bits() => { + dev_info!( + dev, + "M3: core leakage {} replaces the device tree's {}\n", + whole(core), + whole(*c) + ); + *c = core; + } + Some(_) => {} + None => return Err(EINVAL), + } + } + // A SoC whose power model is still a stand-in caps the power target: every operating point's + // power is scaled by the same factor, so the relative powers stay the boot loader's. + let power_cap_mw = match t6031 { + Some(e) => Some(e.power_cap_mw()), + None => soc.power_target_cap_mw, + }; + if let Some(cap) = power_cap_mw { + let max = pwr.max_power_mw; + if max > cap { + for ps in pwr.perf_states.iter_mut() { + ps.pwr_mw = (u64::from(ps.pwr_mw) * u64::from(cap) / u64::from(max)) as u32; + } + pwr.max_power_mw = pwr.perf_states.iter().map(|ps| ps.pwr_mw).max().unwrap_or(cap); + dev_info!( + dev, + "M3: GPU power target capped at {} mW (the device tree's operating points reach {} mW)\n", + pwr.max_power_mw, + max + ); + } + } + let node = dev.of_node().ok_or(ENODEV)?; + let dyncfg = hw::DynConfig { + uat_ttb_base: firmware.resources.regions[0].base, + id: soc_identity(soc, cfg, t6031.and_then(|e| e.revision_id())), + pwr, + firmware_version: node + .get_property::>(c_str!("apple,firmware-version")) + .unwrap_or_default(), + hw_data_a: KVVec::new(), + hw_data_b: KVVec::new(), + hw_globals: KVVec::new(), + }; + let g15 = G15Options { + layout: G15Layout::Runtime, + split_pstates: true, + // The cap is applied to the uploaded objects, for either source (`publish_pstates`). + cap: None, + timestamp_base: Some(crate::agx_memory::TIMESTAMP_RANGE.start), + reference_ppm: true, + runtime_hwdata_b: Some(runtime_hwdata_b), + per_cluster_voltages: soc.per_cluster_voltages, + }; + let c = initdata::InitDataBuilderG15V14_8_3::g15_contents(cfg, &dyncfg, &g15).inspect_err(|e| { + dev_err!(dev, "M3: cannot build InitData from the device tree ({:?})\n", e) + })?; + + let mut images = KVec::new(); + for _ in 0..storage::COUNT { + images.push(None, GFP_KERNEL)?; + } + + // InitData: the magic of the identified firmware, the host-mapped allocations flag and the + // UAT description; the pointers are the constructed record's. + type InitData = raw::InitDataG15V14_8_3<'static>; + let mut root = zeroed(INITDATA)?; + write(&mut root, offset_of!(InitData, ver_info), &firmware.initdata_magic.to_le_bytes())?; + write(&mut root, offset_of!(InitData, unk_2c), &1u32.to_le_bytes())?; + write(&mut root, offset_of!(InitData, uat_page_size), &0x4000u16.to_le_bytes())?; + write(&mut root, offset_of!(InitData, uat_page_bits), &[14, 3])?; + for (i, level) in c.uat_levels.iter().enumerate() { + let at = offset_of!(InitData, uat_level_info) + i * size_of::(); + write(&mut root, at, initdata::raw_bytes(level))?; + } + images[INITDATA] = Some(root); + + // RuntimePointers: only its configuration words; everything else is a pointer. + type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>; + let mut rt = zeroed(RUNTIME_POINTERS)?; + write(&mut rt, offset_of!(RuntimePointers, unk_2d0), &g15.runtime_pointers_2d0().to_le_bytes())?; + write(&mut rt, offset_of!(RuntimePointers, cswitch_hist_ctl), &[0xff])?; + images[RUNTIME_POINTERS] = Some(rt); + + // HwData: HwDataB, then HwDataA. + let mut hwdata = zeroed(HWDATA)?; + write(&mut hwdata, 0, initdata::raw_bytes(&*c.hwdata_b))?; + // The T8122 start experiment's unit masks (`asahi.t8122_unit_mask_a`/`_b`). + if let Some(e) = experiment { + type B = raw::HwDataBG15V14_8_3; + let (a, b) = e.unit_masks(); + write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; + write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; + dev_info!(dev, "M3 G15G start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); + } else if let Some(e) = t6031 { + type B = raw::HwDataBG15V14_8_3; + let (a, b) = e.unit_masks(); + write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; + write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; + dev_info!(dev, "M3 G15C start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); + } + write(&mut hwdata, HWDATA_A, initdata::raw_bytes(&*c.hwdata_a))?; + // The system counter value at InitData creation, the base of the firmware's first power and + // energy interval. + let now: u64; + // SAFETY: reading the architectural counter has no side effects and is allowed at EL1/EL2. + unsafe { core::arch::asm!("mrs {}, cntpct_el0", out(reg) now, options(nomem, nostack, preserves_flags)) }; + write( + &mut hwdata, + HWDATA_A + offset_of!(raw::HwDataAG15V14_8_3, init_timestamp), + &now.to_le_bytes(), + )?; + // The SoC's MTR sensor masks, where its table gives them. + if let Some(m) = soc.mtr_masks { + type A = raw::HwDataAG15V14_8_3; + for at in [offset_of!(A, fast_die0_sensor_mask), offset_of!(A, fast_die0_sensor_mask_2)] { + write(&mut hwdata, HWDATA_A + at, &m.fast_die.to_le_bytes())?; + } + write(&mut hwdata, HWDATA_A + offset_of!(A, fast_die0_sensor_mask_alt), &m.alarm.to_le_bytes())?; + dev_info!( + dev, + "M3: MTR sensor masks: fast-die {:#x}, alarm {:#x}\n", + m.fast_die, + m.alarm + ); + } else if let Some(e) = t6031 { + let (fast, alarm) = e.mtr_masks(dev)?; + type A = raw::HwDataAG15V14_8_3; + for at in [offset_of!(A, fast_die0_sensor_mask), offset_of!(A, fast_die0_sensor_mask_2)] { + write(&mut hwdata, HWDATA_A + at, &fast.to_le_bytes())?; + } + write(&mut hwdata, HWDATA_A + offset_of!(A, fast_die0_sensor_mask_alt), &alarm.to_le_bytes())?; + dev_info!(dev, "M3: MTR sensor masks: fast-die {:#x}, alarm {:#x}\n", fast, alarm); + } + // The second leakage fuse value: the first entry of HwDataA's third leakage table (the first + // table holds the core coefficient, from the builder). + if let Some([_, second]) = fuse { + type A = raw::HwDataAG15V14_8_3; + let at = HWDATA_A + offset_of!(A, cluster_tables) + 2 * size_of::<[u32; 8]>(); + write(&mut hwdata, at, &second.to_bits().to_le_bytes())?; + } + // The SoC's own values for words the shared builder writes otherwise. An armed T6031 copies + // the T8122 words only when asahi.t6031_fw_words=t8122. + let (hwdata_words, globals_words): (&[(usize, u32)], &[(usize, u32)]) = match t6031 { + Some(e) if e.fw_words() => ( + &crate::m3_soc::T8122_HWDATA_WORDS, + &crate::m3_soc::T8122_GLOBALS_WORDS, + ), + Some(_) => (&[], &[]), + None => (soc.hwdata_words, soc.globals_words), + }; + for &(at, value) in hwdata_words { + write(&mut hwdata, at, &value.to_le_bytes())?; + } + // T8122 with the G15 14.8.3 firmware (the only image the runtime accepts there): the 20 + // words of crate::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. + if crate::t8122_start::is_t8122(soc) + && core::ptr::eq(firmware.layout, &crate::m3_firmware::T8122_LAYOUT) + && cfg.gpu_gen == hw::GpuGen::G15 + { + if m3_params::t8122_hwdata_418c() { + let a = hwdata + .get_mut(HWDATA_A..HWDATA_A + size_of::()) + .ok_or(EINVAL)?; + t8122_hwdata::write(a).map_err(|()| EINVAL)?; + dev_info!( + dev, + "M3 G15G: HwDataA +0x418c..+0x41db: wrote the 20 words (asahi.t8122_hwdata_418c=0 leaves them 0)\n" + ); + } else { + dev_info!( + dev, + "M3 G15G: HwDataA +0x418c..+0x41db left 0 (asahi.t8122_hwdata_418c=0)\n" + ); + } + } + // On T8122, log and require the two HwDataB words the firmware's power management depends + // on: it powers the GPU cores up for a job only when +0xa38 and +0xa40 are both nonzero. + if core::ptr::eq(soc, &crate::m3_soc::T8122) { + type B = raw::HwDataBG15V14_8_3; + let a38 = read_u32(&hwdata, offset_of!(B, unk_460))?; + let a40 = read_u32(&hwdata, offset_of!(B, unk_468))?; + dev_info!(dev, "M3: HwDataB +0xa38 {:#x}, +0xa40 {:#x}\n", a38, a40); + if a38 == 0 || a40 == 0 { + dev_err!( + dev, + "M3: HwDataB +0xa38 or +0xa40 is 0; the firmware would not power the GPU cores\n" + ); + return Err(EINVAL); + } + } + fill_io_mappings(dev, cfg, experiment, t6031, io_mappings, iomaps, &mut hwdata)?; + images[HWDATA] = Some(hwdata); + + let mut globals = zeroed(GLOBALS)?; + write(&mut globals, 0, initdata::raw_bytes(&*c.globals))?; + // The gate of the firmware's frequency-feedback cap, per SoC. + let (ut, given) = m3_params::ut_engagement(soc); + write(&mut globals, offset_of!(raw::GlobalsG15V14_8_3, ut_engagement), &ut.to_le_bytes())?; + for &(at, value) in globals_words { + write(&mut globals, at, &value.to_le_bytes())?; + } + if !hwdata_words.is_empty() || !globals_words.is_empty() { + dev_info!( + dev, + "M3: {} HwData and {} Globals words set from the {} table\n", + hwdata_words.len(), + globals_words.len(), + soc.name + ); + } + if ut != 1 || given { + dev_info!( + dev, + "M3: Globals ut_engagement={}{}\n", + ut, + if given { " (asahi.m3_ut_engagement)" } else { "" } + ); + } + if crate::m3_globals_debug::apply(soc.chip_id,m3_params::globals_debug_off(),&mut globals) + .map_err(|_|EINVAL)? { + dev_info!(dev,"M3: T6030 Globals debug word cleared at +0x2c\n"); + } + images[GLOBALS] = Some(globals); + + let mut power = zeroed(GLOBALS_POWER)?; + write(&mut power, 0, initdata::raw_bytes(&*c.power_ctl_block))?; + images[GLOBALS_POWER] = Some(power); + + let mut status = zeroed(STATUS)?; + write(&mut status, 0, initdata::raw_bytes(&*c.status_block))?; + images[STATUS] = Some(status); + + let top = c.tables.primary_state(&dyncfg.pwr, c.tables.len - 1); + dev_info!( + dev, + "M3: device-tree InitData: {} performance states in two voltage-sorted tables from {} device-tree states, top {} MHz at {} mV, SRAM floor {} mV, maximum power {} mW\n", + c.tables.len - 1, + dyncfg.pwr.perf_states.len() - 1, + top.freq_hz / 1_000_000, + top.max_volt_mv(), + dyncfg.pwr.min_sram_microvolt.div_ceil(1000), + dyncfg.pwr.max_power_mw + ); + log_tables(dev, &c.hwdata_b, c.tables.len); + Ok(images) +} + +/// Log the published performance-state tables, one line per entry: the primary table (fastest +/// state per voltage) and the secondary one (slowest), each with the device-tree index of the +/// entry, and the third table. +fn log_tables(dev: &device::Device, b: &raw::HwDataBG15V14_8_3, len: usize) { + let uniform = |row: &[u32; 8]| row.iter().all(|v| *v == row[0]); + for i in 1..len.min(16) { + let aux = &b.aux_ps_3; + let ok = uniform(&b.voltages[i]) && uniform(&b.voltages_sram[i]); + dev_info!( + dev, + "M3: performance state {}: primary dt {} {} MHz core {} mV SRAM {} mV; secondary dt {} {} MHz; third {} MHz core {} mV SRAM {} mV{}\n", + i, + b.unk_arr_0[i], + b.frequencies[i], + b.voltages[i][0], + b.voltages_sram[i][0], + b.unk_arr_0[16 + i], + b.frequencies_2[i], + aux.frequencies[i], + aux.voltages[i][0], + aux.voltages_sram[i][0], + if ok { "" } else { " (columns differ)" } + ); + } +} + +/// HwDataA words that hold the highest usable performance state scaled by 100. +const HWA_MAX_SCALED: [usize; 15] = { + type A = raw::HwDataAG15V14_8_3; + let se = offset_of!(A, unk_e10_0); + [ + offset_of!(A, max_pstate_scaled), + offset_of!(A, max_pstate_scaled_2), + offset_of!(A, max_pstate_scaled_3), + offset_of!(A, max_pstate_scaled_4), + offset_of!(A, max_pstate_scaled_5), + offset_of!(A, max_pstate_scaled_7), + offset_of!(A, max_pstate_scaled_8), + offset_of!(A, max_pstate_scaled_9), + offset_of!(A, max_pstate_scaled_10), + offset_of!(A, max_pstate_scaled_11), + offset_of!(A, max_pstate_scaled_12), + offset_of!(A, max_pstate_scaled_13), + offset_of!(A, max_pstate_scaled_14), + se + offset_of!(raw::HwDataA140Extra, max_pstate_scaled_1), + se + offset_of!(raw::HwDataA140Extra, max_pstate_scaled_2), + ] +}; + +/// HwDataA words that hold the base (lowest settled) performance state scaled by 100. +const HWA_BASE_SCALED: [usize; 4] = { + type A = raw::HwDataAG15V14_8_3; + [ + offset_of!(A, base_pstate_scaled), + offset_of!(A, base_pstate_scaled_2), + offset_of!(A, base_pstate_scaled_3), + offset_of!(A, base_pstate_scaled_4), + ] +}; + +/// Offsets in Globals of the runtime cap words: the second and third power-interface targets and +/// the performance-state cap, each the highest usable state scaled by 100. +const GLOBALS_RUNTIME_CAP: [usize; 3] = { + type G = raw::GlobalsG15V14_8_3; + let power_if_target = offset_of!(G, power_if_target); + [power_if_target + 4, power_if_target + 8, offset_of!(G, perf_state_cap)] +}; + +pub(crate) fn publish_pstates( + dev: &device::Device, + hwdata: &mut Buffer, + globals: &mut Buffer, + p: &PstatePolicy, +) -> Result { + use crate::float::F32; + type A = raw::HwDataAG15V14_8_3; + let a = |offset: usize| HWDATA_A + offset; + let full = 100 * p.table_max; + let base_scaled = hwdata.read_u32(a(offset_of!(A, base_pstate_scaled)))?; + let globals_max = GLOBALS_RUNTIME_CAP; + + let mut expected = true; + for off in HWA_MAX_SCALED { + expected &= hwdata.read_u32(a(off))? == full; + } + for off in HWA_BASE_SCALED { + expected &= hwdata.read_u32(a(off))? == base_scaled; + } + for off in globals_max { + expected &= globals.read_u32(off)? == full; + } + expected &= hwdata.read_u32(a(offset_of!(A, max_pstate_scaled_6)))? == F32::from(full).to_bits(); + let base = base_scaled / 100; + if !expected + || base_scaled % 100 != 0 + || base == 0 + || base > p.safe + || p.boot > p.safe + || p.safe > p.max + { + dev_err!(dev, "M3: unexpected performance-state words in the InitData; not starting the firmware\n"); + return Err(EINVAL); + } + + let ceiling = 100 * p.max; + let max_power = F32::from_bits(hwdata.read_u32(a(offset_of!(A, max_power_1)))?); + let min_duty = hwdata.read_u32(a(offset_of!(A, pwr_min_duty_cycle)))?; + for off in HWA_MAX_SCALED { + hwdata.u32(a(off), ceiling)?; + } + hwdata.u32(a(offset_of!(A, max_pstate_scaled_6)), F32::from(ceiling).to_bits())?; + hwdata.u32(a(offset_of!(A, pwr_pstate_related_k)), (-F32::from(ceiling) / max_power).to_bits())?; + hwdata.u32( + a(offset_of!(A, pwr_pstate_max_dc_offset)), + (min_duty as i32 - ceiling as i32) as u32, + )?; + hwdata.u32( + a(offset_of!(A, boost_state_unk_k)), + (F32::from(p.max - base) / crate::f32!(0.95)).to_bits(), + )?; + for off in globals_max { + globals.u32(off, 100 * p.safe)?; + } + + for off in [offset_of!(A, actual_pstate), offset_of!(A, tgt_pstate)] { + hwdata.u32(a(off), p.boot)?; + } + let settle = base; + dev_info!( + dev, + "M3: firmware performance request: state {} ({} MHz), base state {}, cap state {} ({} MHz) of 1..={}\n", + p.boot, + p.boot_mhz, + settle, + p.safe, + p.safe_mhz, + p.table_max + ); + if p.safe != p.max { + dev_info!( + dev, + "M3: firmware highest state {} ({} MHz); the runtime cap starts at {} ({} MHz)\n", + p.max, + p.max_mhz, + p.safe, + p.safe_mhz + ); + } + Ok(()) +} + +/// Set the runtime cap: the Globals runtime cap words, which the firmware keeps reading while it +/// runs, so the store alone moves the cap, with no message. `cap` must be a state of the +/// published table. +pub(crate) fn publish_runtime_cap(globals: &mut Buffer, cap: u32) -> Result { + if cap == 0 || cap >= 16 { + return Err(EINVAL); + } + for off in GLOBALS_RUNTIME_CAP { + globals.u32(off, 100 * cap)?; + } + crate::agx_memory::publish(); + Ok(()) +} + +/// The integer part of a non-negative, finite F32 given by its bits; None otherwise. +fn f32_bits_to_u32(bits: u32) -> Option { + let exp = (bits >> 23) & 0xff; + if bits >> 31 != 0 || exp == 0xff { + return None; + } + if exp < 127 { + return Some(0); + } + let mant = u64::from((bits & 0x7f_ffff) | 0x80_0000); + let shift = exp - 127; + let value = if shift >= 23 { + mant.checked_shl(shift - 23)? + } else { + mant >> (23 - shift) + }; + u32::try_from(value).ok() +} + +/// One reading of the performance state: SGX+0xe01000, and the HwDataA actual and current +/// state and frequency (MHz, F32 bits) the firmware publishes. +#[derive(Copy, Clone, PartialEq, Eq)] +struct PstateSample { + register: u32, + actual: u32, + current: u32, + freq_bits: u32, +} + +/// Number of lines logging changes of the performance state per boot. +const PSTATE_LOG_LINES: u32 = 64; + +/// Number of lines warning about a reported state above the runtime cap per boot. +const RUNTIME_CAP_WARN_LINES: u32 = 16; + +/// Checks the performance state the firmware reports against the ceiling and the runtime cap, +/// after boot and after every job. +/// +/// The firmware publishes its actual and current state and its frequency in HwDataA. A +/// frequency above the ceiling entry's, or, when no running frequency is reported, a state in +/// the published table above the ceiling, that is still there when read again 1 ms later fails +/// the check. The same above the runtime cap (when it is below the ceiling) is reported to the +/// caller, which decides whether it persists too long (the thermal limit). Values outside the +/// published table are logged and not enforced: they are not states of this table. +/// SGX+0xe01000 is logged next to them but not enforced, as the index space of that register is +/// not established on every board. The published tables list device-tree operating points only, +/// so the check guards the cap, not the voltages. +pub(crate) struct PstateWatch { + policy: PstatePolicy, + last: Option, + lines: u32, + warn_lines: u32, + register_noted: bool, + unknown_noted: bool, + excused_noted: bool, +} + +impl PstateWatch { + pub(crate) fn new(policy: PstatePolicy) -> Self { + PstateWatch { + policy, + last: None, + lines: 0, + warn_lines: 0, + register_noted: false, + unknown_noted: false, + excused_noted: false, + } + } + + fn sample(hwdata: &mut Buffer, register: u32) -> Result { + type A = raw::HwDataAG15V14_8_3; + Ok(PstateSample { + register, + actual: hwdata.read_u32(HWDATA_A + offset_of!(A, actual_pstate))?, + current: hwdata.read_u32(HWDATA_A + offset_of!(A, cur_pstate))?, + freq_bits: hwdata.read_u32(HWDATA_A + offset_of!(A, freq_mhz))?, + }) + } + + /// Frequency of entry `state` of the published table, in MHz. + fn mhz(&self, state: u32) -> u32 { + self.policy.freqs.get(state as usize).copied().unwrap_or(0) + } + + /// The reported frequency in MHz, if it is a running frequency of the published table. + fn freq_mhz(&self, s: &PstateSample) -> Option { + f32_bits_to_u32(s.freq_bits).filter(|mhz| *mhz > 0 && *mhz <= self.policy.table_max_mhz) + } + + /// Whether a reported state lies within the published table but above `cap`. + fn state_above(&self, s: &PstateSample, cap: u32) -> Option<(&'static str, u32)> { + let above = |v: u32| v > cap && v <= self.policy.table_max; + if above(s.actual) { + Some(("actual state", s.actual)) + } else if above(s.current) { + Some(("current state", s.current)) + } else { + None + } + } + + /// The reported value above `cap` that lies within the published table, if any. The + /// frequency is the physical quantity: a state above the cap is excused when the reported + /// frequency is a running frequency within the cap (the state words then use another + /// index space), and counts otherwise. + fn above(&self, s: &PstateSample, cap: u32) -> Option<(&'static str, u32)> { + match self.freq_mhz(s) { + Some(mhz) if mhz > self.mhz(cap) => Some(("frequency (MHz)", mhz)), + Some(_) => None, + None => self.state_above(s, cap), + } + } + + /// Whether a reported value is not a state or frequency of the published table. + fn unknown(&self, s: &PstateSample) -> bool { + let p = &self.policy; + s.actual > p.table_max + || s.current > p.table_max + || f32_bits_to_u32(s.freq_bits).is_none_or(|mhz| mhz > p.table_max_mhz) + } + + fn log(&self, dev: &device::Device, what: &str, s: &PstateSample, cap: u32) { + dev_info!( + dev, + "M3: performance state {}: firmware actual {} current {} frequency {} MHz, sgx+0xe01000 {:#x}; cap state {} ({} MHz)\n", + what, + s.actual, + s.current, + f32_bits_to_u32(s.freq_bits).unwrap_or(u32::MAX), + s.register, + cap, + self.mhz(cap) + ); + } + + /// Read the existing firmware state words and SGX state register without + /// requesting a state, publishing a cap, or enforcing a sample. + pub(crate) fn log_active(&self, dev: &device::Device, hwdata: &mut Buffer, + register: u32, cap: u32, sequence: u32) -> Result { + let s = Self::sample(hwdata, register)?; + dev_info!(dev, + "M3_ACTIVE_POWER sample={} actual={} current={} freq_bits={:#x} sgx={:#x} cap={} cap_mhz={}\n", + sequence, s.actual, s.current, s.freq_bits, s.register, cap, self.mhz(cap)); + Ok(()) + } + + /// Check the reported performance state. `register` reads SGX+0xe01000; `what` names the + /// point of the check for the log; `cap` is the runtime cap the firmware may use now + /// (clamped to the ceiling). Returns ERANGE when the firmware stays above the ceiling; + /// otherwise whether it stays above `cap`, and the highest state it reports (0 if neither + /// state word is a state of the published table). + pub(crate) fn check( + &mut self, + dev: &device::Device, + hwdata: &mut Buffer, + register: impl Fn() -> Result, + what: &str, + cap: u32, + ) -> Result<(bool, u32)> { + let ceiling = self.policy.max; + let cap = cap.clamp(1, ceiling); + let mut s = Self::sample(hwdata, register()?)?; + let mut over = false; + if let Some((field, value)) = self.above(&s, cap) { + if cap == ceiling || self.warn_lines < RUNTIME_CAP_WARN_LINES { + self.warn_lines = self.warn_lines.saturating_add(u32::from(cap != ceiling)); + dev_warn!( + dev, + "M3: firmware reports {} {} above the cap {} ({} MHz) {}; reading again\n", + field, + value, + cap, + self.mhz(cap), + what + ); + } + fsleep(Delta::from_millis(1)); + s = Self::sample(hwdata, register()?)?; + if let Some((field, value)) = self.above(&s, ceiling) { + self.log(dev, what, &s, cap); + dev_err!( + dev, + "M3: firmware {} {} stays above the cap {} ({} MHz); marking the GPU failed\n", + field, + value, + ceiling, + self.mhz(ceiling) + ); + return Err(ERANGE); + } + over = self.above(&s, cap).is_some(); + } + if !self.excused_noted { + if let Some((field, value)) = self.state_above(&s, cap) { + if !over { + self.excused_noted = true; + dev_info!( + dev, + "M3: firmware {} {} is above the cap {} but its frequency is within it {}; not enforced\n", + field, + value, + cap, + what + ); + } + } + } + if !self.unknown_noted && self.unknown(&s) { + self.unknown_noted = true; + dev_info!( + dev, + "M3: firmware performance words outside the published table (states 1..={}, up to {} MHz) {}; not enforced\n", + self.policy.table_max, + self.policy.table_max_mhz, + what + ); + self.log(dev, what, &s, cap); + } + if !self.register_noted && s.register & 0xf > cap { + self.register_noted = true; + dev_info!( + dev, + "M3: sgx+0xe01000 reads {:#x}, above the cap as a table index; logged only\n", + s.register + ); + } + let key = PstateSample { + register: s.register & 0xf, + ..s + }; + if self.last != Some(key) { + if self.lines < PSTATE_LOG_LINES { + self.log(dev, what, &s, cap); + } else if self.lines == PSTATE_LOG_LINES { + dev_info!(dev, "M3: further performance state changes are not logged\n"); + } + self.lines = self.lines.saturating_add(1); + self.last = Some(key); + } + let peak = [s.actual, s.current] + .into_iter() + .filter(|v| *v <= self.policy.table_max) + .max() + .unwrap_or(0); + Ok((over, peak)) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs new file mode 100644 index 0000000000000..5ecfd820b4e4f --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_board.rs @@ -0,0 +1,559 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Board facts for the M3 runtime, read from the hardware and the device tree instead of being +//! fixed to one board. The per-SoC facts are in [`crate::m3_soc`]. + +use kernel::{bindings, c_str, device, device::Core, io::resource::Resource, of, platform, prelude::*, uapi}; + +use crate::m3_resources::{ + reserved_resource, + Region, + Resources, // +}; +use crate::m3_soc::Soc; + +/// Whether the root node's compatible list contains `compatible`. +fn board_is(compatible: &[u8]) -> bool { + let Some(root) = kernel::of::root() else { + return false; + }; + let Ok(board) = root.get_property::>(c_str!("compatible")) else { + return false; + }; + board.split(|b| *b == 0).any(|s| s == compatible) +} + +/// Whether this is a board the M3 runtime is validated on (`soc.validated_boards`). +/// `asahi.m3_expose=auto` registers the render node and `asahi.m3_backend=auto` starts the +/// runtime only on these. +pub(crate) fn runtime_validated_board(soc: &Soc) -> bool { + soc.validated_boards.iter().any(|board| board_is(board)) +} + +/// The number of core slots of `soc` (2 x 10 on T6030, 4 x 10 on T6031). +pub(crate) fn core_slots(soc: &Soc) -> u32 { + soc.cores_per_cluster * soc.clusters +} + +/// Whether the core-enable mask describes a usable GPU of `soc`: at least one core, and no core +/// outside its core slots. The mask is the fused core-enable words from SGX+0xe01500, the second +/// word as bits 32..63 when the SoC has more than 32 core slots. It varies with the SKU (0x6f5fc +/// on a 14-core T6030, 0x7fdff on an 18-core one); absent cores must never be enabled. +pub(crate) fn core_mask_valid(soc: &Soc, mask: u64) -> bool { + let slots = core_slots(soc); + mask != 0 && slots <= 64 && (slots == 64 || mask >> slots == 0) +} + +/// Per-cluster UAPI core masks for a core-enable mask of `soc`: consecutive fields of +/// `cores_per_cluster` bits, cluster 0 lowest, across the word boundary (the packing the +/// identification code uses for every multi-word mask, `regs.rs`). +pub(crate) fn core_masks(soc: &Soc, mask: u64) -> [u32; uapi::DRM_ASAHI_MAX_CLUSTERS as usize] { + let mut masks = [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize]; + let width = soc.cores_per_cluster.min(32); + let field = (1u64 << width) - 1; + for (cluster, slot) in masks.iter_mut().enumerate().take(soc.clusters as usize) { + let shift = cluster as u32 * soc.cores_per_cluster; + *slot = mask.checked_shr(shift).map_or(0, |m| (m & field) as u32); + } + masks +} + +/// The highest GPU frequency in the device tree OPP table, in kHz, or None when the GPU node +/// has no populated OPP table. +pub(crate) fn max_frequency_khz(pdev: &platform::Device) -> Option { + let node = pdev.as_ref().of_node()?; + let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; + let mut max_hz: u64 = 0; + for opp in opps.children() { + if let Ok(hz) = opp.get_property::(c_str!("opp-hz")) { + max_hz = max_hz.max(hz); + } + } + u32::try_from(max_hz / 1000).ok().filter(|khz| *khz != 0) +} + +/// The shape of the performance-state table the device tree's operating points give: states +/// above the off state and the highest frequency in MHz (`t8122_admission::opp_table_shape`). +/// Every cluster of a point must have the same voltage, except on a SoC with per-cluster +/// voltages, where the highest `opp-microvolt` cell orders the point. None without a readable +/// OPP table. +pub(crate) fn opp_table_shape(pdev: &platform::Device, soc: &Soc) -> Option<(u32, u32)> { + let node = pdev.as_ref().of_node()?; + let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; + let mut points = KVec::new(); + for opp in opps.children() { + let hz = opp.get_property::(c_str!("opp-hz")).ok()?; + let microvolt = opp.get_property::>(c_str!("opp-microvolt")).ok()?; + let mv = microvolt.first()?.div_ceil(1000); + if microvolt.len() != soc.clusters as usize + || (!soc.per_cluster_voltages && microvolt.iter().any(|uv| uv.div_ceil(1000) != mv)) + { + return None; + } + // With per-cluster voltages a point is ordered by its highest cluster voltage, as the + // published tables are (`initdata::PStateTables`). + let uv = if soc.per_cluster_voltages { + *microvolt.iter().max()? + } else { + *microvolt.first()? + }; + points.push((hz, uv), GFP_KERNEL).ok()?; + } + crate::t8122_admission::opp_table_shape(points) +} + +/// Register windows the runtime maps on `soc`: name, base, minimum size. The runtime uses the +/// first 16 KiB of `asc` (`m3_device`); the first 16 MiB of `sgx` cover the ID block at +/// +0xd04000, the Fender MMU block at +0xd08000 and the core masks at +0xe01500. +fn reg_windows(soc: &Soc) -> [(&'static CStr, u64, u64); 2] { + [ + (c_str!("asc"), soc.windows.asc, 0x4000), + (c_str!("sgx"), soc.windows.sgx, 0x100_0000), + ] +} + +/// The `reg` cells (two address and two size cells) of the mailbox of `soc`: 16 KiB at its base. +fn expected_mailbox_reg(soc: &Soc) -> [u32; 4] { + [(soc.windows.mailbox >> 32) as u32, soc.windows.mailbox as u32, 0, 0x4000] +} + +/// Reserved regions of the firmware handoff, in `Resources::regions` order. +const REGIONS: [&CStr; 6] = [ + c_str!("ttbs"), + c_str!("pagetables"), + c_str!("handoff"), + c_str!("shared-l2"), + c_str!("fw-text"), + c_str!("fw-data"), +]; + +/// Static reserved-memory nodes that describe a handoff region when the GPU node does not list +/// it under its own name: the bootloader fills these nodes (the UAT regions of the firmware's +/// page-table handoff) whether or not the GPU node references them. The second-level table of +/// the firmware's upper page-table tree ("shared-l2") is reserved this way without being in the +/// GPU node's memory-region list. +const REGION_NODES: [(&CStr, &CStr); 4] = [ + (c_str!("ttbs"), c_str!("/reserved-memory/uat-ttbs")), + (c_str!("pagetables"), c_str!("/reserved-memory/uat-pagetables")), + (c_str!("handoff"), c_str!("/reserved-memory/uat-handoff")), + (c_str!("shared-l2"), c_str!("/reserved-memory/uat-pagetables-l2")), +]; + +/// Properties that describe the loaded firmware's segments on the GPU node. +const SEGMENT_PROPS: [&CStr; 3] = [ + c_str!("apple,m3-handoff-version"), + c_str!("apple,firmware-segment-vas"), + c_str!("apple,firmware-segment-flags"), +]; + +/// Power-management properties of a GPU node whose SoC takes them from the boot loader +/// (`Soc::power_from_boot_loader`), one 32-bit cell each: the ones the T6030 device tree carries +/// as static values, and the shader-engine target. The boot loader copies each from the ADT's GPU +/// node (`apple,X` from `gpu-X`; `apple,se-target` from `gpu-se-tgt`). +/// +/// `apple,idleoff-standby-timer` is not among them: the J613 ADT has no +/// `gpu-idleoff-standby-timer`, so the boot loader adds it only when the ADT has one, and the +/// HwConfig's default applies otherwise. +const BOOT_LOADER_POWER: [&CStr; 40] = [ + c_str!("apple,perf-base-pstate"), + c_str!("apple,min-sram-microvolt"), + c_str!("apple,avg-power-filter-tc-ms"), + c_str!("apple,avg-power-ki-only"), + c_str!("apple,avg-power-kp"), + c_str!("apple,avg-power-min-duty-cycle"), + c_str!("apple,avg-power-target-filter-tc"), + c_str!("apple,fast-die0-integral-gain"), + c_str!("apple,fast-die0-proportional-gain"), + c_str!("apple,perf-boost-ce-step"), + c_str!("apple,perf-boost-min-util"), + c_str!("apple,perf-filter-drop-threshold"), + c_str!("apple,perf-filter-time-constant"), + c_str!("apple,perf-filter-time-constant2"), + c_str!("apple,perf-integral-gain"), + c_str!("apple,perf-integral-gain2"), + c_str!("apple,perf-integral-min-clamp"), + c_str!("apple,perf-proportional-gain"), + c_str!("apple,perf-proportional-gain2"), + c_str!("apple,perf-tgt-utilization"), + c_str!("apple,power-sample-period"), + c_str!("apple,ppm-filter-time-constant-ms"), + c_str!("apple,ppm-ki"), + c_str!("apple,ppm-kp"), + c_str!("apple,pwr-filter-time-constant"), + c_str!("apple,pwr-integral-gain"), + c_str!("apple,pwr-integral-min-clamp"), + c_str!("apple,pwr-min-duty-cycle"), + c_str!("apple,pwr-proportional-gain"), + c_str!("apple,pwr-sample-period-aic-clks"), + c_str!("apple,se-engagement-criteria"), + c_str!("apple,se-filter-time-constant"), + c_str!("apple,se-filter-time-constant-1"), + c_str!("apple,se-inactive-threshold"), + c_str!("apple,se-ki"), + c_str!("apple,se-ki-1"), + c_str!("apple,se-kp"), + c_str!("apple,se-kp-1"), + c_str!("apple,se-reset-criteria"), + c_str!("apple,se-target"), +]; + +/// Check that the boot loader gave a GPU node of `soc` every per-machine property the runtime +/// would otherwise take from a static device tree or a driver default: [`BOOT_LOADER_POWER`], the +/// 64-bit `apple,fast-die0-sensor-mask`, one core and one SRAM leakage coefficient per cluster, +/// the firmware version (three cells, not all zero), and an operating-point table of at least two +/// entries, each with one voltage per cluster and a power, and a frequency after the first. Logs +/// every missing or malformed property. +fn check_boot_loader_power(dev: &device::Device, node: &of::Node, soc: &Soc) -> Result { + let mut bad = 0u32; + let mut expect = |name: &CStr, want: usize, nonzero: bool| { + match node.get_property::>(name) { + Ok(value) if value.len() == want && (!nonzero || value.iter().any(|b| *b != 0)) => {} + Ok(value) => { + dev_info!( + dev, + "M3: not admitted: {:?} is {} bytes, {} expected{}\n", + name, + value.len(), + want, + if nonzero { ", not all zero" } else { "" } + ); + bad += 1; + } + Err(_) => { + dev_info!(dev, "M3: not admitted: the boot loader did not add {:?}\n", name); + bad += 1; + } + } + }; + for name in BOOT_LOADER_POWER { + expect(name, 4, false); + } + expect(c_str!("apple,fast-die0-sensor-mask"), 8, false); + expect(c_str!("apple,core-leak-coef"), 4 * soc.clusters as usize, false); + expect(c_str!("apple,sram-leak-coef"), 4 * soc.clusters as usize, false); + expect(c_str!("apple,firmware-version"), 12, true); + + let mut opps = 0u32; + let mut opps_ok = true; + match node.parse_phandle(c_str!("operating-points-v2"), 0) { + Some(table) => { + for opp in table.children() { + let hz = opp.get_property::(c_str!("opp-hz")).unwrap_or(0); + let volts = opp.get_property::>(c_str!("opp-microvolt")).map_or(0, |v| v.len()); + let power = opp.get_property::(c_str!("opp-microwatt")).is_ok(); + opps_ok &= volts == soc.clusters as usize && power && (opps == 0 || hz != 0); + opps += 1; + } + } + None => opps_ok = false, + } + if !opps_ok || opps < 2 { + dev_info!( + dev, + "M3: not admitted: the operating points are not the boot loader's ({} entries; each needs opp-hz, {} opp-microvolt cells and opp-microwatt)\n", + opps, + soc.clusters + ); + bad += 1; + } + if bad != 0 { + return Err(EINVAL); + } + dev_info!(dev, "M3: boot loader power configuration accepted ({} operating points)\n", opps); + Ok(()) +} + +/// Whether the GPU node lists the memory region `name` in memory-region-names. +fn lists_region(node: &of::Node, name: &CStr) -> bool { + node.get_property::>(c_str!("memory-region-names")) + .is_ok_and(|names| names.split(|b| *b == 0).any(|n| n == name.to_bytes())) +} + +/// The static reserved-memory node that stands in for the handoff region `name`, if the GPU +/// node does not list the region itself and the device tree has such a node. +fn region_node_path(node: &of::Node, name: &CStr) -> Option<&'static CStr> { + if lists_region(node, name) { + return None; + } + REGION_NODES + .iter() + .find(|(region, _)| region.to_bytes() == name.to_bytes()) + .map(|(_, path)| *path) +} + +/// A reference to a device tree node found by path, released on drop. +struct NodeRef(*mut bindings::device_node); + +impl NodeRef { + fn find(path: &CStr) -> Option { + // SAFETY: `path` is a NUL-terminated string; the lookup takes a node reference or + // returns NULL. + let np = unsafe { bindings::of_find_node_opts_by_path(path.as_char_ptr(), core::ptr::null_mut()) }; + (!np.is_null()).then_some(NodeRef(np)) + } +} + +impl Drop for NodeRef { + fn drop(&mut self) { + // SAFETY: the reference was taken by `find`. + unsafe { bindings::of_node_put(self.0) }; + } +} + +/// Look up the handoff region `name` through its static reserved-memory node (see +/// [`REGION_NODES`]). Returns None when the GPU node lists the region itself or the device tree +/// has no such node; otherwise the region, which must be an enabled reserved-memory node the +/// kernel reserved at boot. Returns whether the node is no-map as well. +pub(crate) fn static_region(node: &of::Node, name: &CStr) -> Option> { + let path = region_node_path(node, name)?; + let np = NodeRef::find(path)?; + // SAFETY: `np` holds a node reference for the duration of these read-only queries. + let (available, rmem, nomap) = unsafe { + ( + bindings::of_device_is_available(np.0), + bindings::of_reserved_mem_lookup(np.0), + bindings::of_property_read_bool(np.0, c_str!("no-map").as_char_ptr()), + ) + }; + if !available || rmem.is_null() { + return Some(Err(EINVAL)); + } + // SAFETY: the reserved-memory table entry lives as long as the kernel. + let (base, size) = unsafe { ((*rmem).base, (*rmem).size) }; + if base == 0 || size == 0 || base.checked_add(size).is_none_or(|end| end > 1 << 42) { + return Some(Err(EINVAL)); + } + let raw = bindings::resource { + start: base, + end: base + size - 1, + flags: bindings::IORESOURCE_MEM as _, + ..Default::default() + }; + // SAFETY: Resource is repr(transparent) over Opaque. The descriptor is owned and + // has no pointers into temporary data; its range is checked above. + Some(Ok((unsafe { core::mem::transmute::(raw) }, nomap))) +} + +/// Admit a GPU of `soc` described by the device tree, either statically or by a runtime overlay. +/// +/// The device must be a GPU node of `soc` with the ASC and SGX windows at their addresses +/// (at least as large as the runtime maps), the GPU coprocessor mailbox, the six handoff regions +/// by name (from the reserved-memory registry or no-map overlay nodes), and the firmware segment +/// description. Every refusal is logged. +pub(crate) fn admit(pdev: &platform::Device, soc: &Soc) -> Result { + let dev = pdev.as_ref(); + let refuse = |what: &str, err: Error| { + dev_info!(dev, "M3: not admitted: {}\n", what); + err + }; + let node = dev.of_node().ok_or(ENODEV)?; + + if !board_is(soc.board.as_bytes()) { + dev_info!(dev, "M3: not admitted: not a {} board\n", soc.name); + return Err(ENODEV); + } + let compatible: KVec = node.get_property(c_str!("compatible"))?; + if !compatible.split(|b| *b == 0).any(|s| s == soc.gpu.as_bytes()) { + dev_info!(dev, "M3: not admitted: GPU node is not {}\n", soc.gpu); + return Err(ENODEV); + } + // The runtime runs the firmware that the bootloader loaded, in place: without the + // bootloader's description of its segments there is nothing to admit. + if !lists_region(&node, c_str!("fw-text")) + || !lists_region(&node, c_str!("fw-data")) + || SEGMENT_PROPS.iter().any(|p| node.get_property::>(p).is_err()) + { + return Err(refuse( + "the device tree does not describe the loaded GPU firmware segments (fw-text/fw-data regions and apple,firmware-segment-* properties); the bootloader predates this", + ENODEV, + )); + } + dev_info!(dev, "M3: board compatible accepted\n"); + if soc.power_from_boot_loader { + check_boot_loader_power(dev, &node, soc)?; + } + + for name in REGIONS { + if lists_region(&node, name) { + continue; + } + match region_node_path(&node, name) { + Some(path) if NodeRef::find(path).is_some() => { + dev_info!(dev, "M3: {:?} region: static node {:?}\n", name, path) + } + _ => { + dev_info!(dev, "M3: not admitted: no {:?} memory region\n", name); + return Err(EINVAL); + } + } + } + dev_info!(dev, "M3: memory region list accepted\n"); + + for (name, base, min_size) in reg_windows(soc) { + let res = pdev + .resource_by_name(name) + .ok_or_else(|| refuse("missing register window", EINVAL))?; + if res.start() != base || res.size() < min_size { + dev_info!( + dev, + "M3: not admitted: {:?} window {:#x}+{:#x}, need {:#x}+{:#x} or larger\n", + name, + res.start(), + res.size(), + base, + min_size + ); + return Err(EINVAL); + } + } + dev_info!(dev, "M3: register resources accepted\n"); + + let mboxes: KVec = node.get_property(c_str!("mboxes"))?; + let mbox = (mboxes.len() == 1) + .then(|| node.parse_phandle(c_str!("mboxes"), 0)) + .flatten() + .ok_or_else(|| refuse("expected exactly one mailbox", EINVAL))?; + let compat: KVec = mbox.get_property(c_str!("compatible"))?; + let mbox_reg: KVec = mbox.get_property(c_str!("reg"))?; + let irq_names: KVec = mbox.get_property(c_str!("interrupt-names"))?; + let irqs: KVec = mbox.get_property(c_str!("interrupts"))?; + let cells: u32 = mbox.get_property(c_str!("#mbox-cells"))?; + if !soc.mailbox_compatibles.iter().any(|c| *c == compat.as_slice()) + || mbox_reg.as_slice() != expected_mailbox_reg(soc) + || cells != 0 + || irq_names.as_slice() != b"send-empty\0send-not-empty\0recv-empty\0recv-not-empty\0" + || irqs.as_slice() != soc.mailbox_interrupts + { + return Err(refuse("unexpected GPU mailbox description", EINVAL)); + } + dev_info!(dev, "M3: mailbox resource accepted\n"); + + let mut regions = [Region { base: 0, size: 0 }; 6]; + for (i, name) in REGIONS.iter().enumerate() { + let res = reserved_resource(&node, name).map_err(|e| { + dev_info!(dev, "M3: not admitted: {:?} region unusable ({:?})\n", name, e); + e + })?; + regions[i] = Region { + base: res.start(), + size: res.size(), + }; + } + dev_info!(dev, "M3: reserved resources {:?}\n", regions); + + let version: u32 = node + .get_property(c_str!("apple,m3-handoff-version")) + .map_err(|e| refuse("no firmware handoff description", e))?; + let vas: KVec = node.get_property(c_str!("apple,firmware-segment-vas"))?; + let flags: KVec = node.get_property(c_str!("apple,firmware-segment-flags"))?; + Resources::validate( + version, + regions, + vas.as_slice().try_into().map_err(|_| EINVAL)?, + flags.as_slice().try_into().map_err(|_| EINVAL)?, + ) + .map_err(|e| { + dev_info!(dev, "M3: not admitted: handoff description rejected ({:?})\n", e); + EINVAL + }) +} + +/// Whether the GPU node links to a PMP instance (`apple,pmp`) through which the GPU power vote +/// is cast. Without one, GPU power is left to the power domain. +pub(crate) fn has_pmp_link(pdev: &platform::Device) -> bool { + pdev.as_ref() + .of_node() + .is_some_and(|node| node.get_property::(c_str!("apple,pmp")).is_ok()) +} + +pub(crate) use crate::m3_firmware::{KnownImage, KNOWN_IMAGES, KNOWN_IMAGES_T8122}; +use crate::m3_firmware::image_info; + +fn sha256(data: &[u8]) -> [u8; 32] { + let mut digest = [0u8; 32]; + // SAFETY: `data` and `digest` are valid for the duration of the synchronous call. + unsafe { bindings::sha256(data.as_ptr(), data.len(), digest.as_mut_ptr()) }; + digest +} + +/// Lower-case hex of `bytes`, for logs. +fn hex(bytes: &[u8]) -> [u8; N] { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = [b'0'; N]; + for (i, b) in bytes.iter().take(N / 2).enumerate() { + out[2 * i] = DIGITS[(b >> 4) as usize]; + out[2 * i + 1] = DIGITS[(b & 0xf) as usize]; + } + out +} + +pub(crate) fn identify( + dev: &device::Device, + text: &[u8], + stkg_sha256: &[u8; 32], + images: &'static [KnownImage], +) -> Option<&'static KnownImage> { + let info = image_info(text); + let masked_sha256 = info.as_ref().and_then(|info| { + let mut copy = KVec::new(); + copy.extend_from_slice(text, GFP_KERNEL).ok()?; + copy.get_mut(info.patchbay.clone())?.fill(0); + Some(sha256(©)) + }); + let uuid = info.as_ref().map(|info| info.uuid); + let uuid_hex = hex::<32>(&uuid.unwrap_or([0; 16])); + let stkg_hex = hex::<64>(stkg_sha256); + let masked_hex = hex::<64>(&masked_sha256.unwrap_or([0; 32])); + fn as_str(b: &[u8]) -> &str { + core::str::from_utf8(b).unwrap_or("?") + } + dev_info!( + dev, + "M3: loaded GPU firmware: uuid {} (header {}), text sha256 {} (STKG zeroed), {} (patchbay zeroed)\n", + if uuid.is_some() { as_str(&uuid_hex) } else { "none" }, + if info.as_ref().is_some_and(|i| i.text_size == text.len()) { "ok" } else { "missing or unexpected" }, + as_str(&stkg_hex), + if masked_sha256.is_some() { as_str(&masked_hex) } else { "n/a" } + ); + let image = images.iter().find(|image| image.accepts_runtime(uuid, stkg_sha256)); + match image { + Some(image) => dev_info!(dev, "M3: GPU firmware identified as {}\n", image.name), + None => dev_err!(dev, "M3: loaded GPU firmware is not a known image\n"), + } + image +} + +/// Whether the reserved-memory node behind the GPU memory region `name` is no-map. +/// +/// A region without no-map is part of the kernel's linear map, which maps it write-back; any +/// other CPU mapping of it must use the same memory type, because mismatched aliases of the same +/// memory are not allowed. Only no-map regions may be mapped write-combined. Returns true when +/// the region cannot be looked up, which keeps the caller's own choice. +pub(crate) fn region_is_nomap(node: &of::Node, name: &CStr) -> bool { + if let Some(found) = static_region(node, name) { + return found.map_or(true, |(_, nomap)| nomap); + } + let Ok(names) = node.get_property::>(c_str!("memory-region-names")) else { + return true; + }; + let Some(index) = names.split(|b| *b == 0).position(|n| n == name.to_bytes()) else { + return true; + }; + let Some(region) = node.parse_phandle(c_str!("memory-region"), index) else { + return true; + }; + region.get_property::>(c_str!("no-map")).is_ok() +} + +/// Whether the M3 render node is registered (`asahi.m3_expose`). +/// +/// Userspace drivers pick up any registered render node. Until the M3 runtime is validated on a +/// board, a render node there would let a stock userspace driver submit work the board may not +/// run correctly. `auto` (-1) registers it only on the boards the runtime was validated on. +pub(crate) fn expose_render_node(soc: &Soc) -> bool { + match *crate::module_parameters::m3_expose.value() { + 0 => false, + v if v > 0 => true, + _ => runtime_validated_board(soc), + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs new file mode 100644 index 0000000000000..6661222530a38 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Safety checks for clients of the M3 (G15) render node. +//! +//! Client admission: the M3 GPU runs AGX3 shaders and command streams. Userspace written for +//! the AGX2 GPUs (G13/G14) reads the M3's parameters as those of a G14 and would submit AGX2 +//! streams, which fault the GPU; the M3 backends cannot recover from a fault, so one such client +//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields at +//! the end of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks +//! GET_PARAMS for the whole structure; older userspace asks for less. On G15 a file therefore +//! has to ask for the whole structure before it can create a VM. Every path to the GPU needs a +//! VM, so a refused client can still query the parameters but never reaches the hardware. + +use core::sync::atomic::{ + AtomicU32, + AtomicU64, + Ordering, // +}; + +use kernel::{ + prelude::*, + uapi, +}; + +use crate::{ + driver::AsahiDevice, + agx_uapi::{ + GpuAccess, + GpuAddressSpace, // + }, + mmu, +}; + +/// GET_PARAMS size an AGX3-aware client asks for: the whole global parameter structure. +const AGX3_PARAMS_SIZE: u64 = core::mem::size_of::() as u64; + +/// Refused clients that are logged per boot; later refusals only count. +const REFUSALS_LOGGED: u32 = 16; +static REFUSALS: AtomicU32 = AtomicU32::new(0); + +/// Per-file admission state. +pub(crate) struct ClientGate { + /// Largest size this file has asked GET_PARAMS for the global parameters with. + params_size: AtomicU64, +} + +impl ClientGate { + pub(crate) const fn new() -> Self { + Self { + params_size: AtomicU64::new(0), + } + } + + /// Record a successful GET_PARAMS for the global parameters. + pub(crate) fn note_params(&self, size: u64) { + self.params_size.fetch_max(size, Ordering::Relaxed); + } + + /// Whether this file may create a VM. Only G15 devices are gated. + pub(crate) fn admit_vm(&self, device: &AsahiDevice, flags: u32) -> Result { + let size = self.params_size.load(Ordering::Relaxed); + if crate::g16_profile::client_admitted(device.gpu()?.params()?.gpu_hal_generation, flags, size >= AGX3_PARAMS_SIZE) { + return Ok(()); + } + let refusals = REFUSALS.fetch_add(1, Ordering::Relaxed).saturating_add(1); + if refusals <= REFUSALS_LOGGED { + let task = kernel::current!(); + let comm = task_comm(task); + let len = comm.iter().position(|&b| b == 0).unwrap_or(comm.len()); + let name = core::str::from_utf8(&comm[..len]).unwrap_or("?"); + let last = if refusals == REFUSALS_LOGGED { + " (further refusals are not logged)" + } else { + "" + }; + dev_info!( + device.as_ref(), + "M3: refusing a VM for {}[{}]: it asked GET_PARAMS for {} of the {} parameter bytes, so it cannot know this GPU runs AGX3 command streams; M3-aware userspace is needed{}\n", + name, + task.group_leader().pid(), + size, + AGX3_PARAMS_SIZE, + last + ); + } + Err(Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32))) + } +} + +/// The name of `task`, for log lines only. +pub(crate) fn task_comm(task: &kernel::task::Task) -> [u8; 16] { + let mut name = [0u8; 16]; + // SAFETY: `comm` is a fixed-size array inside the live task. It is read byte by byte with + // volatile loads because a concurrent rename may change it; a torn name is only printed. + unsafe { + let comm = core::ptr::addr_of!((*task.as_ptr()).comm).cast::(); + for (i, byte) in name.iter_mut().enumerate() { + *byte = comm.add(i).read_volatile(); + } + } + name +} + +/// Bytes at the start of a compute control stream whose launch records are checked. +const LAUNCH_WINDOW: usize = 4096; +/// Bytes that must be readable at a launch's program pointer. The pointer is 64-byte aligned, +/// so this never crosses a page. +const PROGRAM_BYTES: u64 = 64; +/// CDM record types (bits 31:29 of a record's first word). +const CDM_LAUNCH: u32 = 0; +const CDM_BARRIER: u32 = 3; + +/// Refuse a compute command whose control stream starts with a launch the GPU cannot fetch. +/// +/// Walks the records at the start of the stream: launches (40 bytes direct, 36 or 24 bytes +/// indirect, told apart by bits 28:27) and barriers (4 bytes). The walk ends at the first link, +/// return, terminator or unknown record, at the end of the stream and after `LAUNCH_WINDOW` +/// bytes. Each launch names its program at `(word1[31:22] << 38) | (word2 << 6)`; that address, +/// and the one read with only word2's low 26 bits when VA[37:32] is set, must be mapped +/// readable in the submitting VM, outside the driver's own mappings, or the command is refused +/// with EINVAL before it is queued. Without the check the GPU would take a translation fault on +/// the fetch, which the M3 backends cannot recover from. +/// +/// This catches malformed or stale pointers, not a hostile client: userspace can rewrite the +/// stream after the check. +pub(crate) fn check_compute_launches( + vm: &mmu::Vm, + space: &A, + base: u64, + end: u64, +) -> Result { + let len = usize::try_from(end.checked_sub(base).ok_or(EINVAL)?) + .unwrap_or(usize::MAX) + .min(LAUNCH_WINDOW); + let mut stream = KVec::from_elem(0u8, len, GFP_KERNEL)?; + if let Err(error) = read_through_cache(vm, base, &mut stream) { + pr_err!( + "M3 compute rejected: control stream {:#x} cannot be read ({:?})\n", + base, + error + ); + return Err(EINVAL); + } + let word = |offset: usize| { + u32::from_le_bytes([ + stream[offset], + stream[offset + 1], + stream[offset + 2], + stream[offset + 3], + ]) + }; + let mut offset = 0; + let mut launch = 0u32; + while offset + 4 <= len { + let head = word(offset); + match head >> 29 { + CDM_LAUNCH => { + if offset + 12 > len { + break; + } + let high = u64::from(word(offset + 4) >> 22) << 38; + let low = u64::from(word(offset + 8)); + // Whether the GPU takes VA[37:32] from word2's top six bits is not + // established, so the program must be mapped under both readings. They + // agree whenever those bits are clear, as in every working launch. + for program in [high | (low << 6), high | ((low & 0x3ff_ffff) << 6)] { + if !space.covers(program, PROGRAM_BYTES, GpuAccess::Read) { + pr_err!( + "M3 compute rejected: launch {} at {:#x} names program {:#x}, which is not mapped readable in this VM\n", + launch, + base + offset as u64, + program + ); + return Err(EINVAL); + } + } + launch += 1; + offset += match (head >> 27) & 3 { + 0 => 40, + 1 => 36, + 2 => 24, + _ => break, + }; + } + CDM_BARRIER => offset += 4, + _ => break, + } + } + Ok(()) +} + +/// Read `out.len()` bytes at `iova` in `vm` through the kernel's linear map. +/// +/// Userspace normally writes command streams through a write-combined mapping, which does not +/// update cache lines the kernel's cacheable alias may hold from an earlier read or a +/// speculative fetch. Each line is cleaned and invalidated before it is read, so the bytes +/// come from memory, as the GPU will fetch them. Cleaning never discards data: a dirty line from +/// a cached mapping is written back first. +/// +/// `Vm::read_mapped_bytes` holds the VM execution lock from translation until the copy, so the +/// pages cannot be unmapped and freed while they are cleaned and read. +fn read_through_cache(vm: &mmu::Vm, iova: u64, out: &mut [u8]) -> Result { + vm.read_mapped_bytes(iova, out, |src, len| clean_invalidate(src as usize, len)) +} + +/// Clean and invalidate the data cache lines covering `start..start + len` to the point of +/// coherency, completing before later loads. +fn clean_invalidate(start: usize, len: usize) { + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1 and has no side effects. + unsafe { + core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, options(nomem, nostack, preserves_flags)); + } + // DminLine: log2 of the smallest data cache line, in 4-byte words. + let line = 4usize << ((ctr >> 16) & 0xf); + let end = start.saturating_add(len); + let mut address = start & !(line - 1); + // SAFETY: the range lies inside one page of the linear map that the caller has mapped. + // Clean-and-invalidate writes dirty lines back before dropping them, so no data is lost. + unsafe { + core::arch::asm!("dsb sy", options(nostack, preserves_flags)); + while address < end { + core::arch::asm!("dc civac, {address}", address = in(reg) address, options(nostack, preserves_flags)); + address += line; + } + core::arch::asm!("dsb sy", options(nostack, preserves_flags)); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs new file mode 100644 index 0000000000000..08114c4d8cc8a --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Completion work owned only by the M3 runtime. + +use kernel::{prelude::*, sync::Arc, workqueue::{self, impl_has_work, new_work, Work, WorkItem}}; +use crate::{driver, m3_drm::Shared}; + +#[pin_data] +pub(crate) struct Completion { + shared: Shared, + #[pin] + work: Work, +} +impl Completion { + pub(crate) fn new(shared: Shared) -> Result> { + Arc::pin_init(try_pin_init!(Self { + shared, + work <- new_work!("M3 completion"), + }), GFP_KERNEL) + } +} +impl_has_work! { impl HasWork for Completion { self.work } } +impl WorkItem for Completion { + type Pointer = Arc; + fn run(owner: Arc) { + use crate::m3_runtime::Service; + // Snapshots after a notification before sleeping for the next one: the completion + // stamps can trail the firmware's event message. + const TRAILING_POLLS: u32 = 4; + let (mut trailing, mut waited_at) = (0, u64::MAX); + loop { + let (outcome, events) = { + let mut guard = owner.shared.lock(); + let Some(runtime) = Option::as_mut(&mut *guard) else { return; }; + let outcome=runtime.service_job(); + let events=runtime.events(); + // Preserve the root-only CPU-overlap diagnostic: disabled waits retain + // the runtime lock so VM/timestamp preparation cannot overlap the GPU. + if let Service::Waiting(messages)=outcome { + if !events.cpu_overlap() {events.wait_past(messages);continue;} + } + (outcome,events) + }; + match outcome { + Service::Idle => break, + Service::Progress => { trailing = 0; waited_at = u64::MAX; } + Service::Waiting(messages) => { + if messages != waited_at { waited_at = messages; trailing = 0; } + if trailing < TRAILING_POLLS { + trailing += 1; + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5)); + } else { + // Until the next firmware event, or a bounded 100 us resnapshot. + events.wait_past(messages); + trailing = 0; + } + } + } + } + } +} +pub(crate) fn queue(dev: &driver::AsahiDevice) { + if let Some(work) = dev.completion.as_ref() { + let _ = workqueue::system_highpri().enqueue(work.clone()); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs new file mode 100644 index 0000000000000..e7469d31b496f --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -0,0 +1,379 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned RTKit 2419 M3 compute graph, populated with a validated user CDM. +use kernel::prelude::*; +use crate::m3_queue_layout as queue; +use crate::m3_state_layout as state; +use crate::{m3_compute_layout as command, m3_compute_sequence as microsequence, + m3_pool_layout::GpuRegion, m3_init_layout::Region}; +use crate::{driver,mmu,agx_memory::{self},m3_memory::Buffer,m3_compute_storage as storage}; +use crate::m3_timeline as timeline; +use storage::{NOTIFIER,PREEMPTION,GPU_CONTEXT,JOB_LIST,QUEUE_SCRATCH,TIMESTAMPS, + TAIL_SCRATCH,MICROSEQUENCE_SCRATCH,Mapping}; +const UMA_POOL:crate::m3_pool::Owners=crate::m3_pool::Owners { + backing:storage::POOL_BACKING,pages:storage::POOL_PAGES,table:storage::POOL_TABLE, + manager:storage::POOL,counter:storage::COUNTER,table_index:2, +}; +// Retain at most four VM views, including the active one, as the render +// graph does. Game and compositor compute jobs otherwise remap the whole +// private arena every time they alternate. Each entry pins its VM binding. +struct CachedViews { + command: mmu::KernelMapping, + views: KVec<(usize, mmu::KernelMapping)>, + // Release cached mappings before returning their ASID lease. + binding: mmu::VmBind, +} +pub(crate) struct Compute { + _client_command: mmu::KernelMapping, + objects: KVec, + command_bytes: KVec, + sequence_bytes: KVec, + _binding: mmu::VmBind, + stats: Region, + sequence: u64, + last_progress_ns: i64, + head: u16, + cached_views: KVec, + slot: usize, + batch_count: usize, + checkpoint: (u64,u16,usize), + /// Firmware-only barrier storage, allocated only for mixed publications. + barriers: Option, + /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). + registers: command::RegisterSet, +} +impl Compute { + pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, vm: &mmu::Vm, + stats: Region, control: crate::agx_compute::Control, + registers: command::RegisterSet) -> Result { + let binding=uat.bind(vm)?; + let mut objects=KVec::new(); + let mut client_command=None; + let mut command_bytes = KVec::new(); + command_bytes.resize(command::SIZE, 0u8, GFP_KERNEL)?; + let mut sequence_bytes = KVec::new(); + sequence_bytes.resize(microsequence::STORAGE, 0u8, GFP_KERNEL)?; + // A private compute aperture, disjoint from the retained render graph. + // Address-bearing graph fields are constructed from their live owners. + for index in 0..storage::COUNT { + let a=storage::batch_allocation(index).map_err(|_|EINVAL)?; + let mut b=match a.mapping { + Mapping::CommandAlias(va)=>Buffer::at(dev,uat.kernel_vm(),Some((uat.kernel_lower_vm(),va)),None,a.size,true)?, + Mapping::Client(va)=>Buffer::at(dev,uat.kernel_vm(),Some((vm,va)),None,a.size,true)?, + Mapping::FirmwareGpuShared=>Buffer::at(dev,uat.kernel_vm(),None,None,a.size,true)?, + Mapping::FirmwareOnly=>Buffer::at(dev,uat.kernel_vm(),None,None,a.size,false)?, + }; + if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);} + objects.push(b,GFP_KERNEL)?; + } + Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, registers)?; + Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0)?; + UMA_POOL.initialize(&mut objects)?; + Self::initialize_state(&mut objects, binding.slot())?; + // Queue references are firmware VAs from live allocations. In contrast + // to render's unpublished seed entries, compute dispatches entry zero + // immediately, so its initial producer cursor remains one. + let address = |index: usize| queue::FirmwareVa::new(objects[index].va()).map_err(|_| EINVAL); + let info = queue::QueueInfo { state: address(storage::QUEUE_STATE)?, ring: address(storage::RING)?, + job_list: address(JOB_LIST)?, gpu_scratch: address(QUEUE_SCRATCH)?, + context: address(GPU_CONTEXT)?, uuid: 0xc10000 }; + let command = address(storage::COMMAND)?; + let mut bytes = [0; queue::INFO_SIZE]; + info.encode(&mut bytes).map_err(|_| EINVAL)?; + objects[storage::QUEUE].write(0, &bytes)?; + let mut state = [0; queue::STATE_SIZE]; + queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 1 } + .encode(&mut state).map_err(|_| EINVAL)?; + objects[storage::QUEUE_STATE].write(0, &state)?; + for index in 0..queue::QUALIFIED_CAPACITY { + let entry = queue::ring_entry(index, queue::QUALIFIED_CAPACITY, &[command]).map_err(|_| EINVAL)?; + objects[storage::RING].write(index as usize * 8, &entry)?; + } + dev_info!(dev.as_ref(),"M3 compute graph: context={} queue={:#x} command={:#x} low={:#x} cdm={:#x}..{:#x} pool={:#x}\n", + binding.slot(),objects[storage::QUEUE].va(),objects[storage::COMMAND].va(), + objects[storage::COMMAND].gpu_va()?,control.base,control.end,objects[storage::POOL].va()); + // The M3 compute stream uses absolute IOTO addresses and does not + // include the USC_EXEC_BASE_CP register. + let mut stream=[0u8;48]; + vm.read_bytes(control.base,&mut stream)?; + crate::cls_dev_dbg!(Compute,dev,"M3 client CDM bytes={:02x?}\n",stream); + agx_memory::publish(); + // Finish the write-combining page-table stores before broadcasting + // the invalidation. + crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers}) + } + fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, + control: crate::agx_compute::Control, sequence: u64, slot: usize, + register_set: command::RegisterSet) -> Result { + let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); + let preemption = &objects[PREEMPTION]; + let value = command::Command { + context, counter: timeline::previous(sequence), notifier: fw(NOTIFIER)?, + preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation(PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, + cdm: command::GpuVa::new(control.base).map_err(|_| EINVAL)?, + cdm_last: command::GpuVa::new(control.end.checked_sub(4).ok_or(EINVAL)?).map_err(|_| EINVAL)?, + gpu_alias: command::GpuVa::new(objects[storage::COMMAND].gpu_va()?+offset(storage::COMMAND)? as u64).map_err(|_| EINVAL)?, + microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: microsequence::LENGTH as u32, + stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?, + stamp_value: timeline::stamp(0xc1000000,sequence), + timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?], + pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false, + register_set, + }; + value.encode(bytes).map_err(|_| EINVAL)?; + objects[storage::COMMAND].write(offset(storage::COMMAND)?, bytes) + } + fn encode_sequence(objects: &mut [Buffer], bytes: &mut [u8], stats: Region, + context: u32, stamp_value: u32, slot: usize) -> Result { + let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); + let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation(owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); + let value = microsequence::Sequence { + command: region(storage::COMMAND)?, stats, notifier: region(NOTIFIER)?, + queue: fw(storage::QUEUE)?, context, generation: 1, pool: fw(storage::POOL)?, + scratch: [fw(MICROSEQUENCE_SCRATCH[0])?, fw(MICROSEQUENCE_SCRATCH[1])?, + fw(MICROSEQUENCE_SCRATCH[2])?, fw(MICROSEQUENCE_SCRATCH[3])?, fw(MICROSEQUENCE_SCRATCH[4])?], + fw_stamp: fw(storage::FW_STAMP)?, stamp_value, + }; + value.encode(bytes).map_err(|_| EINVAL)?; + objects[storage::MICROSEQUENCE].write(offset(storage::MICROSEQUENCE)?, bytes) + } + fn initialize_state(objects: &mut [Buffer], context_slot: u32) -> Result { + let address = |owner: usize| queue::FirmwareVa::new(objects[owner].va()).map_err(|_| EINVAL); + let notification = state::Notifier { threshold: address(storage::EVENT)?, context: context_slot }; + let list_owner = address(JOB_LIST)?; + let mut context = [0; state::CONTEXT_SIZE]; + state::context(&mut context).map_err(|_| EINVAL)?; + objects[GPU_CONTEXT].write(0, &context)?; + let mut notifier = [0; state::NOTIFIER_SIZE]; + notification.encode(&mut notifier).map_err(|_| EINVAL)?; + objects[NOTIFIER].write(0, ¬ifier)?; + let mut list = [0; state::JOB_LIST_SIZE]; + state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; + objects[JOB_LIST].write(0, &list)?; + for (owner, value) in [(storage::COUNTER, 1), (storage::EVENT, 1), (storage::STAMP, timeline::stamp(0xc1000000,0))] { + let mut bytes = [0; state::MANAGER_STATE_SIZE]; + let bytes = bytes.get_mut(..objects[owner].size()).ok_or(EINVAL)?; + state::counter(bytes, value).map_err(|_| EINVAL)?; + objects[owner].write(0, bytes)?; + } + // Fill the preemption buffer with 0xcc, including when a compute + // command is submitted again. + objects[PREEMPTION].fill(0xcc) + } + pub(crate) fn replay(&mut self, uat: &mmu::Uat, vm: &mmu::Vm, + control: crate::agx_compute::Control) -> Result { + if !self.complete()? { return Err(EBUSY); } + // All prior flushed stamps/events and engine idle were checked by + // Runtime before entry. Keep pool/queue ownership, remap only client + // views and patch the context when switching Vulkan processes. + if !self._binding.matches(vm) { + let next = if let Some(index) = self.cached_views.iter().position(|v| v.binding.matches(vm)) { + self.cached_views.remove(index).map_err(|_| EIO)? + } else { + let binding = uat.bind(vm)?; + let mut views = KVec::new(); + for i in 0..storage::COUNT { + if matches!(storage::allocation(i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { + views.push((i,self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; + } + } + let command = self.objects[storage::COMMAND].map_gpu_view(vm)?; + CachedViews {binding, command, views} + }; + // Complete allocations before moving the active views. Eviction + // drops only an idle cached view; no arena storage is duplicated. + let old_views = KVec::with_capacity(next.views.len(), GFP_KERNEL)?; + self.cached_views.reserve(1, GFP_KERNEL)?; + if self.cached_views.len() == 3 { + drop(self.cached_views.remove(0).map_err(|_| EIO)?); + } + let mut old = CachedViews { + binding: core::mem::replace(&mut self._binding, next.binding), + command: core::mem::replace(&mut self._client_command, next.command), + views: old_views, + }; + for (i, view) in next.views { + old.views.push((i, self.objects[i].swap_gpu_view(view)?), GFP_KERNEL)?; + } + self.cached_views.push(old, GFP_KERNEL)?; + } + self.checkpoint=(self.sequence,self.head,self.batch_count); + self.batch_count=0; + if let Err(error)=self.append(control) { + self.abort_batch()?; + return Err(error); + } + Ok(()) + } + /// Append only while the previous batch is fully retired and this one + /// remains unpublished. Firmware serializes each command, including its + /// original completion flush, in this one compute queue. + pub(crate) fn append(&mut self,control:crate::agx_compute::Control)->Result { + if self.batch_count>=storage::SLOTS {return Err(E2BIG);} + self.slot=self.batch_count; + for i in storage::RESET { + let offset=storage::slot_offset(i,self.slot).map_err(|_|EINVAL)?; + let size=storage::allocation(i).map_err(|_|EINVAL)?.size; + self.objects[i].fill_range(offset,size,if i==PREEMPTION {0xcc} else {0})?; + } + self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?; + self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?; + Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers)?; + let stamp=self.stamp(); + Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot)?; + self.objects[storage::EVENT].u32(0,self.sequence as u32)?; + self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; + let address=self.objects[storage::COMMAND].va()+storage::slot_offset(storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; + self.objects[storage::RING].u64(usize::from(self.head)*8,address)?; + self.head=(self.head+1)%queue::QUALIFIED_CAPACITY as u16; + self.batch_count+=1; + self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; + agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + Ok(()) + } + /// Roll back producer state before any SubmitQueue publication. Old ring + /// entries beyond the restored cursor are unreachable and get overwritten. + pub(crate) fn abort_batch(&mut self)->Result { + (self.sequence,self.head,self.batch_count)=self.checkpoint; + self.slot=self.batch_count.saturating_sub(1); + self.objects[storage::EVENT].u32(0,self.sequence as u32)?; + self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; + self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; + agx_memory::publish();Ok(()) + } + fn offset(&self,owner:usize)->Result { + storage::slot_offset(owner,self.slot).map_err(|_|EINVAL) + } + /// Same cache-generation contract as M4: StartCompute and the persistent + /// notifier must use the same tag before the firmware flushes user stamps. + /// RTKit 2419: StartCompute.counter1 +0x34; JobMeta.flush_stamps +0x804. + pub(crate) fn prepare_completion(&mut self,coalesce:bool)->Result { + let sequence=self.offset(storage::MICROSEQUENCE)?; + let command=self.offset(storage::COMMAND)?; + self.objects[storage::MICROSEQUENCE].u32(sequence+microsequence::GENERATION,0)?; + self.objects[storage::COMMAND].u32(command+command::FLUSH_STAMPS,u32::from(!coalesce))?; + agx_memory::publish();Ok(()) + } + pub(crate) fn set_user_timestamps(&mut self,addresses:[u64;2])->Result { + let offset=self.offset(storage::COMMAND)?+command::USER_TIMESTAMPS; + self.objects[storage::COMMAND].u64(offset,addresses[0])?; + self.objects[storage::COMMAND].u64(offset+8,addresses[1])?; + let pointer=if addresses==[0,0] {0} else {self.objects[storage::COMMAND].va()+offset as u64}; + let sequence=self.offset(storage::MICROSEQUENCE)?; + for start in [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] { + self.objects[storage::MICROSEQUENCE].u64(sequence+start+microsequence::USER_TIMESTAMP_POINTER,pointer)?; + } + agx_memory::publish();Ok(()) + } + pub(crate) fn set_attachments(&mut self,attachments:&crate::agx_attachments::Attachments)->Result { + let sequence=self.offset(storage::MICROSEQUENCE)?; + self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::ATTACHMENTS,&attachments.0)?; + self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::HAS_ATTACHMENTS,&[u8::from(attachments.count()!=0)])?; + agx_memory::publish();Ok(()) + } + fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } + pub(crate) fn completion_dependency(&self)->Result { + Ok(crate::m3_sync_layout::CompletionDependency { + stamp:queue::FirmwareVa::new(self.objects[storage::FW_STAMP].va()).map_err(|_|EINVAL)?, + value:self.stamp(),event:2, + }) + } + pub(crate) fn reserve_barriers(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat)->Result { + if self.barriers.is_none() { + self.barriers=Some(Buffer::at(dev,uat.kernel_vm(),None,None,storage::SLOTS*64,false)?); + } + Ok(()) + } + /// Insert a dependency before the last appended, still unpublished CDM. + /// Its command ordinal and timestamp slot are unchanged; only the ring + /// gains an entry. The whole batch must retire before this storage is reused. + pub(crate) fn insert_dependency(&mut self,dependency:crate::m3_sync_layout::CompletionDependency)->Result { + if self.batch_count==0 || self.slot>=storage::SLOTS || dependency.event!=1 { + return Err(EINVAL); + } + let value=self.stamp(); + let barriers=self.barriers.as_mut().ok_or(EINVAL)?; + let offset=self.slot*64; + let mut bytes=[0;crate::m3_sync_layout::BARRIER_SIZE]; + crate::m3_sync_layout::Barrier { + stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event, + self_value:value,uuid:0xc10000,dependency:crate::m3_sync_layout::Dependency::PreviousPass, + }.encode(&mut bytes).map_err(|_|EINVAL)?; + barriers.write(offset,&bytes)?; + let address=barriers.va()+offset as u64; + let (slots,next)=queue::insert_before_last(self.head,queue::QUALIFIED_CAPACITY).map_err(|_|EINVAL)?; + let command=self.objects[storage::COMMAND].va()+self.offset(storage::COMMAND)? as u64; + self.objects[storage::RING].u64(usize::from(slots[0])*8,address)?; + self.objects[storage::RING].u64(usize::from(slots[1])*8,command)?; + self.head=next; + self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; + agx_memory::publish();crate::mem::sync();Ok(()) + } + pub(crate) fn gpu_ns(&mut self)->Result { + let mut total=0; + for slot in 0..self.batch_count {total+=self.slot_gpu_ns(slot)?;} + Ok(total) + } + pub(crate) fn slot_gpu_ns(&mut self,slot:usize)->Result { + if slot>=self.batch_count {return Err(EINVAL);} + let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; + let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; + Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + } + /// A batch span includes gaps between its compute commands, unlike gpu_ns. + /// Read only after stamps, queues, events and idle state prove retirement. + pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { + if self.batch_count == 0 {return Err(EINVAL);} + Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, + self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) + } + pub(crate) fn head(&self) -> u16 { self.head } + pub(crate) fn ordinal(&self)->u64 {self.sequence} + pub(crate) fn first(&self) -> bool { self.sequence == self.batch_count as u64 } + pub(crate) fn progress(&mut self, dev: &driver::AsahiDevice) -> Result { + // Report real completions by elapsed time as well as sequence count; + // batches need not land on a multiple of 128. + if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + let now = ::ktime_get(); + if self.last_progress_ns != 0 && now - self.last_progress_ns < 1_000_000_000 + && self.sequence != 1 && self.sequence % 128 != 0 + && !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + self.last_progress_ns = now; + dev_info!(dev.as_ref(), "M3_COMPUTE_RETIRED sequence={} context={} stamp={:#x}/{:#x} queue={}/{} head={} idle=1 cache_flush=1\n", + self.sequence, self._binding.slot(), self.objects[storage::STAMP].read_u32(0)?, + self.objects[storage::FW_STAMP].read_u32(0)?, self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?, + self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?, self.head); + Ok(()) + } + + pub(crate) fn queue(&self)->u64 {self.objects[storage::QUEUE].va()} + pub(crate) fn complete(&mut self)->Result { + Ok(self.objects[storage::STAMP].read_u32(0)?==self.stamp() + && self.objects[storage::FW_STAMP].read_u32(0)?==self.stamp() + && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?==u32::from(self.head) + && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?==u32::from(self.head)) + } + pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + for off in (0..512).step_by(128) { + let mut raw=[0u8;128]; self.objects[storage::COMMAND].read(off,&mut raw)?; + dev_info!(dev.as_ref(),"M3 post-command {:#x} {:02x?}\n",off,raw); + } + let stamp=self.objects[storage::STAMP].read_u32(0)?; + let fw=self.objects[storage::FW_STAMP].read_u32(0)?; + let done=self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?; + let read=self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?; + dev_info!(dev.as_ref(),"M3 compute: stamps={:#x}/{:#x} done={} read={}\n",stamp,fw,done,read); + let mut record=[0u8;0x84];self.objects[storage::POOL].read(0,&mut record)?; + dev_info!(dev.as_ref(),"M3 compute pool={:02x?}\n",record); + let mut roots=[(0,0);64];self._binding.vm().context_roots(&mut roots)?; + dev_info!(dev.as_ref(),"M3 context {} roots={:x?}, bootstrap={:x?}\n",self._binding.slot(),roots[self._binding.slot() as usize],roots[0]); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs new file mode 100644 index 0000000000000..59277aaefb681 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned RTKit 2419.140.12 configuration with named initialization references. +use kernel::prelude::*; +use crate::m3_init_layout as init; +use crate::m3_init_storage as storage; +use storage::{REGION_A,RUNTIME_POINTERS,HARDWARE_DATA,UNKNOWN_PAIR,FWLOG_PAYLOAD, + UNKNOWN_C0,UNKNOWN_C1,UNKNOWN_C3,GLOBALS,GLOBALS_POWER,CONTROL_REGION,RUNTIME_FLAGS, + FW_CONTROL_STATE,FW_CONTROL_RING}; +use crate::{driver,agx_memory::{self},m3_memory::Buffer,mmu,pgtable::prot}; +kernel::static_assert!(core::mem::size_of::() == init::fwlog::ENTRY_SIZE); +kernel::static_assert!(core::mem::size_of::() == init::fwlog::PAYLOAD_SIZE); +kernel::static_assert!(::SUB_CHANNELS == init::fwlog::CHANNELS); + +/// The root, runtime-pointer and firmware-control records, encoded for the owners +/// `region` describes. Config uploads them for its own allocations; the device-tree +/// InitData checks encode them for the fixed allocation table before GPU access. +pub(crate) struct Records { + pub(crate) root: [u8; init::ROOT_SIZE], + pub(crate) runtime: [u8; init::RUNTIME_SIZE], + pub(crate) control: KVec, +} +impl Records { + pub(crate) fn encode(region: impl Fn(usize) -> Result) -> Result { + let root = init::Root { region_a: region(REGION_A)?, runtime: region(RUNTIME_POINTERS)?, + globals: region(GLOBALS)?, new_a: region(RUNTIME_FLAGS)?, + control: region(CONTROL_REGION)?, power: region(GLOBALS_POWER)? }; + let mut channels = [init::Channel { state: region(0)?, ring: region(1)? }; 17]; + for (index, channel) in channels.iter_mut().enumerate() { + *channel = init::Channel { state: region(index * 2)?, ring: region(index * 2 + 1)? }; + } + let runtime = init::RuntimePointers { hardware: region(HARDWARE_DATA)?, + unknown_pair: region(UNKNOWN_PAIR)?, fwlog_payload: region(FWLOG_PAYLOAD)?, + unknown_c0: region(UNKNOWN_C0)?, unknown_c1: region(UNKNOWN_C1)?, + unknown_c3: region(UNKNOWN_C3)?, channels }; + let control = init::ControlRegion { channel: init::Channel { + state: region(FW_CONTROL_STATE)?, ring: region(FW_CONTROL_RING)? } }; + let mut root_bytes = [0; init::ROOT_SIZE]; + root.encode(&mut root_bytes).map_err(|_| EINVAL)?; + let mut runtime_bytes = [0; init::RUNTIME_SIZE]; + runtime.encode(&mut runtime_bytes).map_err(|_| EINVAL)?; + // The control/status region is 50 KiB; use temporary heap storage, + // never a kernel-stack array. All three owners precede firmware startup. + let mut control_bytes = KVec::new(); + control_bytes.resize(init::CONTROL_SIZE, 0u8, GFP_KERNEL)?; + control.encode(&mut control_bytes).map_err(|_| EINVAL)?; + Ok(Self { root: root_bytes, runtime: runtime_bytes, control: control_bytes }) + } +} + +pub(crate) struct Config { + objects: KVec, + #[cfg(CONFIG_DEV_COREDUMP)] + fault_reserve: Option, + _iomaps: KVec, + pub(crate) completed_events: u64, + pstates: crate::m3_adt_config::PstateWatch, + thermal: crate::m3_thermal::Governor, + request_top: u32, + request_value: u32, + request_checked: bool, + request_check_at: Option>, + request_writes: u64, + request_observed: u32, +} +impl Config { + /// Snapshot owned RAM only. Firmware can still update these buffers; + /// per-record time intervals explicitly describe sequential observations. + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) fn fault_snapshot(&mut self, dev: &driver::AsahiDevice, primary: Error, gpu_pending: bool) -> Result { + let mut dump = self.fault_reserve.take().ok_or(ENOMEM)?; + dump.begin(0); + let mut cause = [0u8; 16]; + cause[..4].copy_from_slice(&1u32.to_le_bytes()); + cause[4..8].copy_from_slice(&primary.to_errno().to_le_bytes()); + cause[8..16].copy_from_slice(&u64::from(gpu_pending).to_le_bytes()); + dump.record("host-first-error", 0, cause.len(), |out| { + out.copy_from_slice(&cause); Ok(()) + })?; + for (name, index) in [ + ("root", storage::ROOT), ("runtime", RUNTIME_POINTERS), + ("hardware", HARDWARE_DATA), ("globals", GLOBALS), + ("control", CONTROL_REGION), ("flags", RUNTIME_FLAGS), + // The power-controller block, and the host-to-firmware device-control and + // firmware-control queues: their state words show whether the firmware read a + // queued control (read index) and what it was sent. + ("power", GLOBALS_POWER), + ("devctl-state", storage::DEVICE_CONTROL * 2), + ("devctl-ring", storage::DEVICE_CONTROL * 2 + 1), + ("fwctl-state", storage::FW_CONTROL_STATE), + ("fwctl-ring", storage::FW_CONTROL_RING), + ("event-state", storage::EVENT * 2), + ("event-ring", storage::EVENT * 2 + 1), + ("fwlog-state", storage::FIRMWARE_LOG * 2), + ("fwlog-ring", storage::FIRMWARE_LOG * 2 + 1), + ("fwlog-payload", FWLOG_PAYLOAD), + ("trace-state", storage::TRACE * 2), + ("trace-ring", storage::TRACE * 2 + 1), + ] { + let buffer = &mut self.objects[index]; + dump.record(name, buffer.va(), buffer.size(), |out| buffer.read(0, out))?; + } + dump.publish(dev.as_ref()) + } + + pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, firmware: &crate::m3_firmware::Firmware, + contents: &crate::m3_adt_config::Contents) -> Result { + contents.check_for_upload()?; + let mut objects = KVec::new(); + let mut iomaps = KVec::new(); + for index in 0..storage::COUNT { + let mut a=storage::allocation(index).map_err(|_|EINVAL)?; + let image=contents.initial(index,&a)?; + if a.initial!=storage::Initial::Zero && image.len()!=a.size {return Err(EINVAL);} + // The SoC may place the HwData object elsewhere and have it mapped cacheable + // (`Soc::hwdata_object`); the records and the upload check use the live address. + let object=if index==HARDWARE_DATA {contents.hwdata_object} else {None}; + if let Some(o)=object {a.address=o.address;} + let mut b=match object { + Some(o) if o.cached=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address),a.size, + prot::PROT_FW_PRIV_RW,prot::PROT_GPU_SHARED_RW)?, + _=>Buffer::at(dev,uat.kernel_vm(),None,Some(a.address),a.size,a.gpu_shared)?, + }; + b.write(0,image)?;objects.push(b,GFP_KERNEL)?; + } + for io in contents.iomaps { + // Read-write unless the contents say otherwise (only the T8122 start experiment). + let protection=if contents.read_only_slots==0 {prot::PROT_FW_MMIO_RW} + else {crate::t8122_start::mmio_prot(contents.read_only_slots,io.slot)}; + let map=uat.kernel_vm().map_io(io.address,io.physical.try_into()?,io.size,protection)?; + let owned=init::Region::new(map.iova(),io.size).map_err(|_|EINVAL)?; + let field=io.pointer_field().map_err(|_|EINVAL)?; + objects[HARDWARE_DATA].u64(field,io.pointer(owned).map_err(|_|EINVAL)?)?; + iomaps.push(map,GFP_KERNEL)?; + } + // This field is a physical address owned by the reserved GPU region, + // unlike the firmware virtual references in the IOMapping records. + objects[HARDWARE_DATA].u64(storage::GPU_REGION_PHYSICAL,firmware.resources.regions[0].base)?; + Self::initialize_records(&mut objects)?; + // The root record carries the G15 14.8.3 InitData version, which is every T6030 + // image's. Only the T8122 start experiment gives the firmware another one + // (`asahi.t8122_initdata_version`). + if firmware.initdata_magic!=crate::m3_firmware::G15_V14_8_3_INITDATA { + objects[storage::ROOT].u64(0,firmware.initdata_magic)?; + } + // Vertex/tessellation loops need not retire a primitive at every + // firmware progress poll. The inherited interval of 10 falsely + // declares a valid 35.7 ms TA job stuck on J514S. Keep progress + // detection enabled, with a finite interval of 100 firmware polls. + // This is independent of the host job bound and 60-second watchdog. + // RTKit 2419 copies Globals+0x980 into its TA poll threshold at + // text+0x6514; text+0xd710 checks it before sampling engine progress. + const TA_PROGRESS_INTERVAL: usize = 0x980; + if objects[GLOBALS].read_u32(TA_PROGRESS_INTERVAL)? != 10 { return Err(EINVAL); } + objects[GLOBALS].u32(TA_PROGRESS_INTERVAL, 100)?; + dev_info!(dev.as_ref(), "M3: firmware TA progress-check interval=100\n"); + // RTKit 2419 +2de48..2dec8 accepts user timestamp stores only + // within the 64 MiB arena at HwDataB+28. Match map_timestamp(). + objects[HARDWARE_DATA].u64(0x28,agx_memory::TIMESTAMP_RANGE.start)?; + // RTKit 2419 HwDataA follows the 0x4580-byte HwDataB. Request the + // boot performance state through firmware (asahi.m3_boot_pstate) and cap + // the states it may use (asahi.m3_max_pstate), retaining the voltage + // tables, power/thermal ceilings and minimum operating state. + // The bring-up image initialized actual/target and every base to 1 + // (338 MHz), so short desktop jobs never left that startup state. + // The uploaded table is the SoC's (`Soc::pstates`): its highest state at +0xb58 and that + // state's frequency in the table at +0xb5c (8 and 1380 MHz on T6030). + let (states, top_mhz) = contents.table; + if objects[HARDWARE_DATA].read_u32(0xb58)? != states + || objects[HARDWARE_DATA].read_u32(0xb5c + states as usize * 4)? != top_mhz + { + return Err(EINVAL); + } + let (hwdata, globals) = objects.split_at_mut(GLOBALS); + crate::m3_adt_config::publish_pstates(dev.as_ref(), &mut hwdata[HARDWARE_DATA], &mut globals[0], &contents.pstates)?; + // Firmware ktrace is for explicit detailed diagnostics. Normal games + // still drain error/events, but need not generate trace records for + // every firmware operation on the command-processing core. + objects[RUNTIME_FLAGS].u32(0,u32::from(crate::debug::debug_enabled( + crate::debug::DebugFlags::SubmitTiming)))?; + crate::m3_adt_config::check_upload(dev.as_ref(), firmware, contents.iomaps, &mut objects)?; + agx_memory::publish(); + let thermal = crate::m3_thermal::Governor::new(dev.as_ref(), &contents.pstates)?; + #[cfg(CONFIG_DEV_COREDUMP)] + let fault_reserve = match crate::agx_fault::Dump::new_m3() { + Ok(dump) => Some(dump), + Err(error) => { + dev_warn!(dev.as_ref(), "M3: fault snapshot reserve unavailable: {:?}\n", error); + None + } + }; + Ok(Self { objects, _iomaps: iomaps, completed_events: 0, + #[cfg(CONFIG_DEV_COREDUMP)] fault_reserve, + pstates: crate::m3_adt_config::PstateWatch::new(contents.pstates), thermal, + request_top:contents.table.0,request_value:0,request_checked:false, + request_check_at:None,request_writes:0,request_observed:0 }) + } + fn initialize_records(objects: &mut [Buffer]) -> Result { + let records = Records::encode(|index: usize| { + init::Region::new(objects[index].va(), objects[index].size()).map_err(|_| EINVAL) + })?; + objects[storage::ROOT].write(0, &records.root)?; + objects[RUNTIME_POINTERS].write(0, &records.runtime)?; + objects[CONTROL_REGION].write(0, &records.control) + } + pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } + pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDevice)->Result { + let value=self.objects[GLOBALS].read_u32(crate::m3_globals_debug::OFFSET)?; + dev_info!(dev.as_ref(),"M3_GLOBALS_DEBUG off={} value={} request_level={}\n", + u8::from(crate::m3_params::globals_debug_off()),value,crate::m3_params::consistent_perf()); + Ok(()) + } + pub(crate) fn performance_request(&mut self,chip:u32,busy:bool)->Result { + let Some(value)=crate::m3_consistent_perf::request(chip, + crate::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; + if value==0 && self.request_value==0 {return Ok(());} + let now=kernel::time::Instant::::now(); + let elapsed=self.request_check_at.map(|last|(now-last).as_nanos()); + if !crate::m3_consistent_perf::refresh_due(value,self.request_value,elapsed) { + return Ok(()); + } + type P=crate::fw::initdata::raw::PowerCtlBlock; + let at=core::mem::offset_of!(P,consistent_perf_state); + if !self.request_checked { + let mut flag=[0]; + self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut flag)?; + if flag[0]!=0 || self.objects[GLOBALS_POWER].read_u32(at)?!=0 {return Err(EINVAL);} + self.request_checked=true; + } + let observed=self.objects[GLOBALS_POWER].read_u32(at)?; + self.request_observed=observed; + if observed!=value { + self.objects[GLOBALS_POWER].packed_request(at,value)?; + agx_memory::publish(); + self.request_writes=self.request_writes.saturating_add(1); + } + self.request_value=value; + self.request_check_at=Some(now); + Ok(()) + } + /// Move the runtime cap by temperature (the thermal limit), then check the performance + /// state the firmware reports against the ceiling and the runtime cap. Called after boot and + /// after every job, by the job thread. + pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDevice, device: &crate::m3_device::Device, what: &str) -> Result { + let now = kernel::time::Instant::::now(); + if let Some(cap) = self.thermal.update(dev.as_ref(), now) { + crate::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; + } + let allowed = self.thermal.allowed(now); + let (over, peak) = self.pstates.check(dev.as_ref(), &mut self.objects[HARDWARE_DATA], || device.pstate_register(), what, allowed)?; + self.thermal.note_check(dev.as_ref(), now, over, peak)?; + self.thermal.busy(now); + Ok(()) + } + pub(crate) fn stats_region(&self) -> Result { + let owner = &self.objects[HARDWARE_DATA]; + init::Region::new(owner.va(), owner.size()).map_err(|_| EINVAL) + } + + pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDevice, + device: &crate::m3_device::Device, + sequence: u32) -> Result { + let cap = self.thermal.allowed(kernel::time::Instant::::now()); + self.pstates.log_active(dev.as_ref(), &mut self.objects[HARDWARE_DATA], + device.pstate_register()?, cap, sequence)?; + type P=crate::fw::initdata::raw::PowerCtlBlock; + type G=crate::fw::initdata::raw::GlobalsG15V14_8_3; + let request=self.objects[GLOBALS_POWER].read_u32(core::mem::offset_of!(P,consistent_perf_state))?; + let cap_word=self.objects[GLOBALS].read_u32(core::mem::offset_of!(G,perf_state_cap))?; + let power_if=core::mem::offset_of!(G,power_if_target); + let target1=self.objects[GLOBALS].read_u32(power_if+4)?; + let target2=self.objects[GLOBALS].read_u32(power_if+8)?; + let mut override_active=[0]; + self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut override_active)?; + dev_info!(dev.as_ref(),"M3_PERF_REQUEST sample={} level={} request={} cap_word={} target1={} target2={} writes={} observed={} override_active={}\n", + sequence,crate::m3_params::consistent_perf(),request,cap_word,target1,target2, + self.request_writes,self.request_observed,override_active[0]); + Ok(()) + } + + pub(crate) fn render_pb(&mut self) -> Result { + self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3_render::PB_PAGES).map_err(|_|EINVAL)?) + } + pub(crate) fn submit_queue(&mut self, kind:usize, queue:u64, head:u16,event:u8,new:bool)->Result { + if kind>2 {return Err(EINVAL);} + let state=kind*2;let ring=state+1; + let w=self.objects[state].read_u32(0x20)?; + if w>=256 || self.objects[state].read_u32(0)?!=w { return Err(EBUSY); } + let mut message=[0u8;0x18]; + message[8..16].copy_from_slice(&queue.to_le_bytes()); + message[16..20].copy_from_slice(&(kind as u32).to_le_bytes()); + message[20..22].copy_from_slice(&head.to_le_bytes()); + message[22]=event;message[23]=u8::from(new); // New queue; caller selected the event slot. + self.objects[ring].write(w as usize*0x18,&message)?; + agx_memory::publish(); + self.objects[state].u32(0x20,(w+1)%256)?; + agx_memory::publish(); + Ok(()) + } + /// Whether the firmware has consumed every message on pipe `kind` (0 TA, 1 3D, 2 compute). + pub(crate) fn pipe_free(&mut self, kind:usize) -> Result { + if kind>2 {return Err(EINVAL);} + Ok(self.objects[kind*2].read_u32(0)? == self.objects[kind*2].read_u32(0x20)?) + } + pub(crate) fn pipes_idle(&mut self) -> Result { + for state in [0,2,4] { + if self.objects[state].read_u32(0)? != self.objects[state].read_u32(0x20)? { + return Ok(false); + } + } + Ok(true) + } + pub(crate) fn ready(&mut self) -> Result { + // NewRegionB+4 is an input initialized to 1, not an acknowledgement. + // These are firmware-written transitions present in compute-15's + // pre-submit snapshot; both start at zero in the exported schema. + Ok(self.objects[RUNTIME_POINTERS].read_u32(0x314)? == 0xabcdabcd + && self.objects[RUNTIME_FLAGS].read_u32(4)? == 1) + } + pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDevice) -> Result { + let magic=self.objects[RUNTIME_POINTERS].read_u32(0x314)?; + let state=self.objects[RUNTIME_POINTERS].read_u32(0x2fc)?; + let a=self.objects[RUNTIME_FLAGS].read_u32(4)?; + let b=self.objects[RUNTIME_FLAGS].read_u32(0x14)?; + let c=self.objects[CONTROL_REGION].read_u32(0x45c4)?; + dev_info!(dev.as_ref(),"M3 firmware readiness: magic={:#x} runtime={} newA={}/{} newB={}\n",magic,state,a,b,c); + Ok(()) + } + /// The firmware's halt counter and its halted and resume flags. + pub(crate) fn halt_state(&mut self) -> Result<(u64,u32,u32)> { + let control=&mut self.objects[CONTROL_REGION]; + Ok((control.read_u64(0x4580)?,control.read_u32(0x4590)?,control.read_u32(0x45a0)?)) + } + /// Let a halted firmware continue: clear halted, then set resume. These + /// words keep the halt/resume layout of earlier Apple GPU firmware. + pub(crate) fn resume_halted(&mut self) -> Result { + let control=&mut self.objects[CONTROL_REGION]; + control.u32(0x4590,0)?; + agx_memory::publish(); + control.u32(0x45a0,1)?; + agx_memory::publish(); + Ok(()) + } + /// Read-only snapshot of the firmware's recovery words in the control + /// region (InitData+0xb0): the halt counter and the halted/resume flags + /// at +0x4580/+0x4590/+0x45a0, the firmware's own recovery information + /// at +0x40a0, and the details it leaves for a recovery event at +0x44a0. + /// Nothing is acknowledged, cleared or resumed here. + pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDevice, events: u64) -> Result { + let control=&mut self.objects[CONTROL_REGION]; + let halt_count=control.read_u64(0x4580)?; + let halted=control.read_u32(0x4590)?; + let resume=control.read_u32(0x45a0)?; + let info=[control.read_u32(0x40a0)?,control.read_u32(0x40a4)?]; + let mut packet=[0u8;0x14]; + control.read(0x44a0,&mut packet)?; + dev_info!(dev.as_ref(),"M3 firmware recovery state: halt_count={} halted={} resume={} info={:#x}/{:#x} packet={:02x?} completed_events={} event_messages={}\n", + halt_count,halted,resume,info[0],info[1],packet,self.completed_events,events); + Ok(()) + } + /// Read-only: log the event ring entries from the host's read index to the firmware's write + /// index (at most 16), as `drain` reads them. Nothing is consumed or acknowledged, and the + /// completion count is not changed. + pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDevice) -> Result { + const SIZE: usize = 0x38; + const COUNT: usize = 256; + let r = self.objects[storage::EVENT*2].read_u32(0)? as usize; + let w = self.objects[storage::EVENT*2].read_u32(0x20)? as usize; + dev_info!(dev.as_ref(), "M3 event ring: read={} write={}\n", r, w); + if r >= COUNT || w >= COUNT { return Err(EIO); } + let mut index = r; + for _ in 0..16 { + if index == w { break; } + let mut raw = [0u8; SIZE]; + self.objects[storage::EVENT*2+1].read(index * SIZE, &mut raw)?; + dev_info!(dev.as_ref(), "M3 event ring entry {} kind={} bytes={:02x?}\n", + index, u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]), raw); + index = (index + 1) % COUNT; + } + Ok(()) + } + pub(crate) fn control(&mut self, opcode: u32) -> Result { + if !matches!(opcode, 0x13 | 9) { return Err(EINVAL); } + let w = self.objects[storage::DEVICE_CONTROL*2].read_u32(0x20)?; + let r = self.objects[storage::DEVICE_CONTROL*2].read_u32(0)?; + if w >= 256 || r != w { return Err(EBUSY); } + let mut bytes = [0; 0x38]; + bytes[..4].copy_from_slice(&opcode.to_le_bytes()); + self.objects[storage::DEVICE_CONTROL*2+1].write(w as usize * 0x38, &bytes)?; + agx_memory::publish(); + let next = (w + 1) % 256; + self.objects[storage::DEVICE_CONTROL*2].u32(0x20, next)?; + agx_memory::publish(); + Ok(next) + } + pub(crate) fn control_done(&mut self, next: u32) -> Result { + Ok(self.objects[storage::DEVICE_CONTROL*2].read_u32(0)? == next && self.objects[storage::DEVICE_CONTROL*2].read_u32(0x10)? == next) + } + pub(crate) fn drain(&mut self, dev: &driver::AsahiDevice) -> Result { + // Event, trace, stats and six independent firmware log rings. + for (state, ring, size, count, channels) in [(storage::EVENT*2,storage::EVENT*2+1,0x38,256,1), + (storage::TRACE*2,storage::TRACE*2+1,0x38,512,1), + (storage::STATS*2,storage::STATS*2+1,0x40,256,1), + (storage::FIRMWARE_LOG*2,storage::FIRMWARE_LOG*2+1,init::fwlog::ENTRY_SIZE,init::fwlog::SLOTS,init::fwlog::CHANNELS)] { + for channel in 0..channels { + let base = channel * 0x30; + let mut r = self.objects[state].read_u32(base)? as usize; + let w = self.objects[state].read_u32(base+0x20)? as usize; + if r >= count || w >= count { return Err(EIO); } + let previous_read = r; + while r != w { + if (state==storage::TRACE*2 && self.completed_events<4) || state==storage::FIRMWARE_LOG*2 { + let mut raw=[0u8;0xd8]; + self.objects[ring].read((channel*count+r)*size,&mut raw[..size])?; + dev_info!(dev.as_ref(),"M3 fw-channel {} {:02x?}\n",state,&raw[..size]); + if state == storage::FIRMWARE_LOG*2 && self.objects[ring].read_u32((channel*count+r)*size)? == 2 { + let index = self.objects[ring].read_u64((channel*count+r)*size+8)?; + let offset = init::fwlog::payload_offset(channel,index).map_err(|_| EIO)?; + let mut payload = [0u8;init::fwlog::PAYLOAD_SIZE]; + self.objects[FWLOG_PAYLOAD].read(offset,&mut payload)?; + dev_info!(dev.as_ref(),"M3 fwlog payload {} {:02x?}\n",channel,payload); + } + } + if state == storage::EVENT*2 { + let kind = self.objects[ring].read_u32(r*size)?; + if kind == 1 { + let flags=self.objects[ring].read_u64(r*size+4)?; + self.completed_events+=u64::from((flags&7).count_ones()); + } + if kind == 0 || kind == 4 { + let mut raw=[0u8;0x38];self.objects[ring].read(r*size,&mut raw)?; + dev_err!(dev.as_ref(), "M3 firmware error event {} bytes={:02x?}\n", kind,raw); + return Err(EIO); + } + } + r = (r+1) % count; + } + // Publish actual consumption only, as RxChannel::get does. + // Empty polls need no write to firmware's shared state line. + if r != previous_read { self.objects[state].u32(base, r as u32)?; } + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs new file mode 100644 index 0000000000000..1c2f0b1e8866d --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Owned M3 register apertures. The platform +//! power domain owns TVM/PMGR and stays on across probe/remove. Firmware and +//! queues must be stopped before this object is released. + +use kernel::{ + bindings, c_str, + device::Core, + devres::Devres, + error::to_result, + io::{ + mem::IoMem, + Io, + }, + platform, + prelude::*, + sync::aref::ARef, + time::{delay::fsleep, Delta, Instant, Monotonic}, +}; + +use crate::m3_firmware::Firmware; +use crate::m3_soc::Soc; + +const ASC_CPU_CONTROL: usize = 0x44; +const ASC_CPU_RUN: u32 = 1 << 4; + +pub(crate) struct Device { + dev: ARef, + asc: Pin>>>, + sgx: Pin>>, + firmware: Firmware, + core_mask: u64, + soc: &'static Soc, + power_vote: bool, +} + +impl Device { + /// `experiment` is the T8122 start experiment's values, on an armed T8122 only: it decides + /// the SGX setup write, which the T8122 table lacks. `t6031` is the same for an armed T6031. + pub(crate) fn new(pdev: &platform::Device, firmware: Firmware, soc: &'static Soc, + experiment: Option<&crate::t8122_start::Experiment>, + t6031: Option<&crate::t6031_start::Experiment>) -> Result { + // Map ASC before taking a vote. The larger SGX aperture contains + // this control window and the separate mailbox provider, so it is + // mapped without a conflicting claim over those child resources. + // Only the 16 KiB control window is used. Some device trees describe + // the ASC block as a larger range that also covers the mailbox, which + // the mailbox driver has already claimed, so do not claim the range. + let asc = KBox::pin_init( + pdev.io_request_by_name(c_str!("asc")) + .ok_or(EINVAL)? + .iomap_sized::<0x4000>(), + GFP_KERNEL, + )?; + dev_info!(pdev.as_ref(), "M3: ASC aperture mapped\n"); + let sgx = KBox::pin_init( + pdev.io_request_by_name(c_str!("sgx")) + .ok_or(EINVAL)? + .iomap(), + GFP_KERNEL, + )?; + dev_info!(pdev.as_ref(), "M3: SGX aperture mapped\n"); + let pmp = crate::m3_board::has_pmp_link(pdev); + let t8122 = crate::t8122_start::is_t8122(soc); + let t6031_soc = crate::t6031_start::is_t6031(soc); + if pmp && !t8122 && !t6031_soc { + dev_err!(pdev.as_ref(), "M3: optional apple,pmp GPU link is unsupported\n"); + return Err(ENOTSUPP); + } + if t8122 && !pmp { + dev_err!(pdev.as_ref(), "M3 G15G: current14 GPU requires its PMP supplier link\n"); + return Err(ENODEV); + } + if pmp { + // SAFETY: the consumer is live; the managed link pins its declared PMP supplier. + to_result(unsafe { bindings::apple_pmp_link_device(pdev.as_ref().as_raw()) })?; + } else { + dev_info!(pdev.as_ref(), "M3: no apple,pmp link; GPU power is left to its power domain\n"); + } + if asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + dev_err!( + pdev.as_ref(), + "M3 {}: ASC already running; refusing to take ownership\n", + soc.gpu_name + ); + return Err(EBUSY); + } + let mut device = Self { + dev: pdev.into(), + asc, + sgx, + firmware, + core_mask: 0, + soc, + power_vote: false, + }; + + if pmp { + // SAFETY: the linked current14 T8122 PMP owns AGX logical device 5. The bridge + // waits for firmware readiness and the power-command acknowledgement. + // An unacknowledged request may still have enabled AGX. Keep ownership so a + // failed acquisition releases it only after the ASC-stopped check in Drop. + device.power_vote = true; + to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 1) })?; + if t6031_soc { + dev_info!(pdev.as_ref(), "M3 G15C: PMP GPU device 5 power acknowledged\n"); + } else { + dev_info!(pdev.as_ref(), "M3 G15G: PMP GPU device 5 power acknowledged\n"); + } + } + + let registers = device.sgx.access(pdev.as_ref())?; + let version = registers.try_read32(0xd04000)?; + let counts = registers.try_read32(0xd04010)?; + // The core-enable words: the second only on a SoC with more than 32 core slots. + let mut core_mask = u64::from(registers.try_read32(0xe01500)?); + if crate::m3_board::core_slots(soc) > 32 { + core_mask |= u64::from(registers.try_read32(0xe01504)?) << 32; + } + dev_info!(pdev.as_ref(), "M3 {}: power acknowledged, version={:#010x} counts={:#010x} core-mask={:#x}, firmware={}\n", + soc.gpu_name, version, counts, core_mask, device.firmware.version()); + // These words identify the qualified configuration of this SoC. The + // fused core mask varies with SKU; absent cores must never be enabled. + if version & soc.id.version_mask != soc.id.version + || counts & soc.id.counts_mask != soc.id.counts + || !crate::m3_board::core_mask_valid(soc, core_mask) + { + return Err(ENODEV); + } + let setup = match experiment { + Some(e) => { + match e.sgx_setup() { + Some((offset, value)) => dev_warn!(pdev.as_ref(), + "M3 G15G start: SGX setup write: SGX+{:#x} = {:#x} (asahi.t8122_sgx_setup=t6030)\n", offset, value), + None => dev_info!(pdev.as_ref(), + "M3 G15G start: no SGX setup write (asahi.t8122_sgx_setup=none)\n"), + } + e.sgx_setup() + } + None => match t6031 { + Some(e) => { + match e.sgx_setup() { + Some((offset, value)) => dev_warn!( + pdev.as_ref(), + "M3 G15C start: SGX setup write: SGX+{:#x} = {:#x}\n", + offset, + value + ), + None => dev_info!(pdev.as_ref(), "M3 G15C start: no SGX setup write\n"), + } + e.sgx_setup() + } + None => Some(soc.sgx_setup.ok_or(ENODEV)?), + }, + }; + if let Some((setup_offset, setup_value)) = setup { + registers.try_write32(setup_value, setup_offset)?; + } + device.core_mask = core_mask; + Ok(device) + } + + pub(crate) fn core_mask(&self) -> u64 { self.core_mask } + + /// The SoC table this device was admitted with. + pub(crate) fn soc(&self) -> &'static Soc { self.soc } + + pub(crate) fn check_idle(&self)->Result { + self.check_idle_parts(true,true) + } + /// [`Self::check_idle`] with its engine-busy (`busy`) and fault-bank (`faults`) register + /// accesses selectable (`asahi.m3_retire_mmio`); with neither it touches no GPU register. + pub(crate) fn check_idle_parts(&self,busy:bool,faults:bool)->Result { + if !busy && !faults {return Ok(());} + let sgx=self.sgx.try_access().ok_or(ENODEV)?; + if busy && (sgx.try_read64(0xc020)? | sgx.try_read64(0xc120)?) & 1 != 0 { + return Err(EBUSY); + } + if !faults {return Ok(());} + let selector=sgx.try_read64(0xd800)?; + let mut fault=0; + // One fault bank (selected through 0xd800) per GPU cluster: banks 0 and 1 on the + // two-cluster T6030, bank 0 alone on the single-cluster T8122. Never select a bank past + // the SoC's clusters. + for bank in 0..u64::from(self.soc.clusters) { + sgx.try_write64(bank,0xd800)?;fault|=sgx.try_read64(0xd8c0)?; + } + sgx.try_write64(selector,0xd800)?; + if fault!=0 {return Err(EIO);} Ok(()) + } + + pub(crate) fn check_drm(&self, pdev: &platform::Device) -> Result { + use kernel::dma::{Device as _, DmaMask}; + // SAFETY: This is the admitted M3 GPU's 42-bit physical DMA capability. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; + crate::mmu::check_handoff_guard()?; + dev_info!(pdev.as_ref(), "M3 {}: GPU handoff lock checked.\n", self.soc.gpu_name); + for _ in 0..2 { + let drm: ARef = kernel::drm::Device::new( + pdev.as_ref(), crate::driver::AsahiData::new(pdev, None, true))?; + if drm.gpu().is_ok() { return Err(EIO); } + let mut object = crate::gem::new_kernel_object(&drm, 0x4000)?; + object.vmap()?.memset(0); + drop(object); + drop(drm); + } + dev_info!(pdev.as_ref(), "M3 {}: DRM data initialized; backend absent, device unregistered\n", self.soc.gpu_name); + Ok(()) + } + + /// The raw performance-state register (SGX+0xe01000). + pub(crate) fn pstate_register(&self) -> Result { + self.sgx.try_access().ok_or(ENODEV)?.try_read32(0xe01000) + } + + /// Read-only snapshot. Do not select or acknowledge fault banks here; + /// the firmware owns those registers until recovery is ported. + pub(crate) fn log_engine_state(&self, vm: &crate::mmu::Vm) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let pstate = sgx.try_read32(0xe01000)? & 0xf; + if pstate == 0 { + dev_info!(self.dev.as_ref(), "M3 {}: GPU is powered down; skipping engine/MMU register snapshot\n", self.soc.gpu_name); + return Ok(()); + } + dev_info!(self.dev.as_ref(), "M3 {}: reading engine/MMU snapshot at pstate={}\n", self.soc.gpu_name, pstate); + let selector=sgx.try_read64(0xd800)?; + for bank in [0,1,0x100,0x40000,0x40001,0x80000,0xc0000] { + sgx.try_write64(bank,0xd800)?; + let address=sgx.try_read64(0xd8c8)?; + let fault=sgx.try_read64(0xd8c0)?; + dev_info!(self.dev.as_ref(),"M3 bank {} fault={:#x} address_word={:#x} address={:#x}\n",bank,fault,address,address<<6); + if fault&1!=0 { + dev_info!(self.dev.as_ref(),"M3 fault mapping bank={} address={:#x} translation={:?}\n",bank,address<<6,vm.translate_iova(address<<6)); + } + } + sgx.try_write64(selector,0xd800)?; + // Read-only per-core service/debug snapshot, using the same + // core-selector protocol as the idle check. + let service=sgx.try_read32(0xa010)?; + let debug=sgx.try_read32(0xa000)?; + let mut cores=u64::from(sgx.try_read32(0xe01500)?); + if crate::m3_board::core_slots(self.soc) > 32 { + cores |= u64::from(sgx.try_read32(0xe01504)?) << 32; + } + for core in 0..crate::m3_board::core_slots(self.soc).min(64) { + if cores & (1u64< &Firmware { &self.firmware } + + pub(crate) fn require_stopped(&self, pdev: &platform::Device) -> Result { + if self.asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + return Err(EBUSY); + } + Ok(()) + } + + pub(crate) fn start_asc(&self, pdev: &platform::Device) -> Result { + self.require_stopped(pdev)?; + let asc = self.asc.access(pdev.as_ref())?; + asc.write32(asc.read32(ASC_CPU_CONTROL) | ASC_CPU_RUN, ASC_CPU_CONTROL); + Ok(()) + } + + pub(crate) fn stop_asc(&self) -> Result { + let asc = self.asc.try_access().ok_or(ENODEV)?; + asc.write32(asc.read32(ASC_CPU_CONTROL) & !ASC_CPU_RUN, ASC_CPU_CONTROL); + let start = Instant::::now(); + while asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + if start.elapsed() >= Delta::from_millis(100) { return Err(ETIMEDOUT); } + fsleep(Delta::from_micros(10)); + } + Ok(()) + } + +} + +impl Drop for Device { + fn drop(&mut self) { + if !self.power_vote { + return; + } + // ASC and firmware queues must be stopped before the inner GPU vote is relinquished. + match self.asc.try_access() { + Some(asc) if asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN == 0 => {} + _ => { + dev_err!(self.dev.as_ref(), "M3 G15G: cannot prove ASC stopped; retaining PMP vote\n"); + return; + } + } + // SAFETY: the managed consumer link keeps the PMP bound through device removal. + if let Err(error) = to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 0) }) { + dev_err!(self.dev.as_ref(), "M3 G15G: PMP GPU power release failed: {:?}\n", error); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs new file mode 100644 index 0000000000000..25e65c6599084 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M3 through the common Asahi GEM, GPUVM and render-node frontend. +use core::ops::Range; +use kernel::{device::Core, drm, new_mutex, platform, prelude::*, sync::{Arc, Mutex}}; +use crate::{alloc, driver, drm_gpu::{DrmGpu, DrmGpuParams}, gem, gpu, hw, mmu, queue}; + +pub(crate) type Shared = Arc>>; +struct ProgressView(Arc); +impl kernel::debugfs::Writer for ProgressView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (generation, completed, last_ns, healthy) = self.0.progress_snapshot(); + writeln!(f, "version=1 generation_ns={} completed={} last_completion_ns={} healthy={}", + generation, completed, last_ns, u8::from(healthy)) + } +} +// Separate endpoint preserves the strict completion-v1 ABI for old guards. +struct ActivityView(Arc); +impl kernel::debugfs::Writer for ActivityView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (generation, epoch, healthy) = self.0.activity_snapshot(); + writeln!(f, "version=1 generation_ns={} epoch={} pending={} healthy={}", + generation, epoch, epoch & 1, u8::from(healthy)) + } +} + +struct TimingView(Arc); +impl kernel::debugfs::Writer for TimingView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (generation, _, _, healthy) = self.0.progress_snapshot(); + self.0.timing.write(f, generation, healthy) + } +} + +// Root-only writable debugfs policy; no mutable module-parameter references. +// Keep ownership independent of the runtime lock and remove the file on stop. +struct CompletionView(Arc); +impl kernel::debugfs::Writer for CompletionView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); + writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", + us, irqs, sleeps, timers, preparations) + } +} +impl kernel::debugfs::Reader for CompletionView { + fn read_from_slice(&self, reader: &mut kernel::uaccess::UserSliceReader) -> Result { + let len = reader.len(); + if len == 0 || len > 8 { return Err(EINVAL); } + let mut bytes = [0u8; 8]; + reader.read_slice(&mut bytes[..len])?; + let text = core::str::from_utf8(&bytes[..len]).map_err(|_| EINVAL)?; + let us = text.trim().parse::().map_err(|_| EINVAL)?; + self.0.set_wait_us(us) + } +} + +struct SubmitOverlapView(Arc); +impl kernel::debugfs::Writer for SubmitOverlapView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "{}", u8::from(self.0.cpu_overlap())) + } +} +impl kernel::debugfs::Reader for SubmitOverlapView { + fn read_from_slice(&self, reader: &mut kernel::uaccess::UserSliceReader) -> Result { + let len=reader.len(); + if len==0 || len>2 {return Err(EINVAL);} + let mut bytes=[0u8;2];reader.read_slice(&mut bytes[..len])?; + let text=core::str::from_utf8(&bytes[..len]).map_err(|_|EINVAL)?.trim(); + match text { + "0" => self.0.set_cpu_overlap(false), + "1" => self.0.set_cpu_overlap(true), + _ => return Err(EINVAL), + } + Ok(()) + } +} + +pub(crate) struct Registered { + registration: Pin>>>>, + shared: Shared, + health: Arc, + _progress: Pin>>, + _activity: Pin>>, + _memory: Pin>>, + _timing: Pin>>, + _completion: Pin>>, + _overlap: Pin>>, +} +impl Registered { + pub(crate) fn start(pdev: &platform::Device, soc: &'static crate::m3_soc::Soc) -> Result { + let resources = crate::m3_resources::from_device(pdev, soc).inspect_err(|_| { + crate::t8122_start::not_admitted(pdev.as_ref(), soc); + crate::t6031_start::not_admitted(pdev.as_ref(), soc); + })?; + dev_info!(pdev.as_ref(), "M3: resource admission complete\n"); + // An armed T8122 or T6031 start supplies the values its table leaves out. Any other SoC, + // or an unarmed T8122 or T6031, needs a complete table. + let experiment = crate::t8122_start::arm(pdev.as_ref(), soc)?; + let t6031 = crate::t6031_start::arm(pdev.as_ref(), soc)?; + if experiment.is_none() && t6031.is_none() { + soc.require_complete(pdev.as_ref())?; + } + let experiment = experiment.as_ref(); + let t6031 = t6031.as_ref(); + let refused = |stage: &'static str| { + move |e: &Error| { + crate::t8122_start::refused(pdev.as_ref(), experiment, stage, *e); + crate::t6031_start::refused(pdev.as_ref(), t6031, stage, *e); + } + }; + let firmware = crate::m3_firmware::identify_loaded(pdev, soc, resources, experiment, t6031) + .inspect_err(refused("the firmware identity"))?; + dev_info!(pdev.as_ref(), "M3: firmware identity accepted\n"); + // InitData source and contents, before any GPU register access. + let contents = crate::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment, t6031) + .inspect_err(refused("the InitData contents"))?; + let device = crate::m3_device::Device::new(pdev, firmware, soc, experiment, t6031) + .inspect_err(refused("the GPU registers and identity"))?; + device.check_drm(pdev).inspect_err(refused("the DRM setup"))?; + let core_mask = device.core_mask(); + let max_frequency_khz = 1000 * contents.pstates.reported_max_mhz(); + let mut runtime = crate::m3_runtime::Runtime::new(pdev, device, contents)?; + runtime.boot(pdev)?; + dev_info!(pdev.as_ref(), "M3: runtime switches: asahi.m3_unlocked_wait={} asahi.m3_timeout_nohang={} resume-after-fault={}\n", + u8::from(crate::m3_params::unlocked_wait()), u8::from(crate::m3_params::timeout_nohang()), + u8::from(crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault))); + let drm = runtime.drm(); + let health = runtime.health(); + // Health holds no device references. The read-only file owns an Arc + // and is removed with registration; teardown latches failure first. + let directory = kernel::debugfs::Dir::new(c"asahi-m3"); + let progress = KBox::pin_init(directory.read_only_file(c"progress", + ProgressView(health.clone())), GFP_KERNEL)?; + let activity = KBox::pin_init(directory.read_only_file(c"activity", + ActivityView(health.clone())), GFP_KERNEL)?; + let memory = KBox::pin_init(directory.read_only_file(c"memory", + crate::agx_memory_stats::View), GFP_KERNEL)?; + let timing = KBox::pin_init(directory.read_only_file(c"timing", + TimingView(health.clone())), GFP_KERNEL)?; + let completion = KBox::pin_init(directory.read_write_file(c"completion_wait", + CompletionView(runtime.completion_wait())), GFP_KERNEL)?; + let overlap = KBox::pin_init(directory.read_write_file(c"submit_overlap", + SubmitOverlapView(runtime.completion_wait())), GFP_KERNEL)?; + let shared = Arc::pin_init(new_mutex!(Some(runtime)), GFP_KERNEL)?; + if !drm.completion.populate(crate::m3_completion::Completion::new(shared.clone())?) { + return Err(EBUSY); + } + let owner = Self { registration: KBox::pin_init(new_mutex!(None), GFP_KERNEL)?, shared: shared.clone(), health: health.clone(), _progress: progress, _activity: activity, _memory: memory, _timing: timing, _completion: completion, _overlap: overlap }; + let scheduler=Arc::new(drm::sched::Scheduler::new(drm.as_ref(),4,8,0,3000,kernel::c_str!("asahi_m3_sched"))?,GFP_KERNEL)?; + let backend: Arc = Arc::new(Backend { shared, health, scheduler, ids: gpu::SequenceIDs::default(), + core_mask, max_frequency_khz, soc }, GFP_KERNEL)?; + if !drm.gpu.populate(crate::drm_gpu::Backend::M3(backend)) { return Err(EBUSY); } + if crate::m3_board::expose_render_node(soc) { + *owner.registration.lock() = Some(drm::driver::Registration::new(&drm, 0)?); + dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); + } else { + dev_info!(pdev.as_ref(), "M3: firmware ready; render node not registered on this board (asahi.m3_expose=1 registers it)\n"); + } + Ok(owner) + } + pub(crate) fn stop(&self) { + self.health.mark_failed(); + drop(self.registration.lock().take()); + drop(self.shared.lock().take()); + } +} +impl Drop for Registered { fn drop(&mut self) { self.stop(); } } +struct Backend { shared: Shared, health: Arc, ids: gpu::SequenceIDs, + scheduler:Arc>, core_mask: u64, max_frequency_khz: u32, + soc: &'static crate::m3_soc::Soc } +impl DrmGpu for Backend { + fn init(&self) -> Result { if self.is_crashed() { Err(ENODEV) } else { Ok(()) } } + fn ids(&self) -> &gpu::SequenceIDs { &self.ids } + fn is_crashed(&self) -> bool { !self.health.healthy() } + fn update_globals(&self) {} + + fn supports_vm_status(&self) -> bool { true } + fn supports_scheduled_queues(&self)->bool {true} + fn submission_error(&self) -> i32 { if self.is_crashed() { EIO.to_errno() } else { 0 } } + fn params(&self) -> Result { + let soc = self.soc; + let masks = crate::m3_board::core_masks(soc, self.core_mask); + Ok(DrmGpuParams { gpu_generation: hw::GpuGen::G15 as u32, + gpu_variant: soc.gpu_variant as u32, gpu_revision: soc.gpu_revision as u32, + chip_id: soc.chip_id, num_dies: 1, num_clusters_total: soc.clusters, + num_cores_per_cluster: soc.cores_per_cluster, + core_masks: masks, max_frequency_khz: self.max_frequency_khz, usc_generation: 3, + gpu_hal_generation: hw::GpuHalGeneration::Legacy as u32, + max_commands_per_submission: crate::file::MAX_COMMANDS_PER_SUBMISSION }) + } + fn user_range(&self) -> Result> { Ok(mmu::UAT_PGSZ as u64..0x2ff_ffff_8000) } + fn unknown_page(&self) -> Result { Ok(0x2ff_ffff_8000) } + fn base_clock_hz(&self) -> u32 { + let value: u64; + // SAFETY: read-only architectural counter frequency. + unsafe { core::arch::asm!("mrs {x}, CNTFRQ_EL0", x=out(reg) value) }; + value as u32 + } + fn new_vm(&self, range: Range) -> Result { + Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.new_vm(self.ids.vm.next(), range) + } + fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, + _ualloc_priv: Arc>, priority: u32, + usc_exec_base: u64) -> Result> { + Ok(KBox::new(crate::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, + self.soc.features.fragment_dependency)?,GFP_KERNEL)?) + } + fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { + self.shared.lock().as_ref().ok_or(ENODEV)?.map_timestamp(bo,range) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs new file mode 100644 index 0000000000000..9ae509ad959c4 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +use crate::m3_resources as agx_resources; + +/// A GPU firmware image the runtime can identify. +#[derive(Debug)] +pub(crate) struct KnownImage { + /// Human-readable identity, for logs. + pub(crate) name: &'static str, + uuid: Option<[u8; 16]>, + stkg_sha256: Option<[u8; 32]>, + /// The validated InitData magic (InitData+0), or `None` until runtime admission is supported. + pub(crate) initdata_magic: Option, +} + +impl KnownImage { + /// Runtime admission requires a validated InitData version and all recorded identities. + pub(crate) fn accepts_runtime(&self, uuid: Option<[u8; 16]>, digest: &[u8; 32]) -> bool { + self.initdata_magic.is_some() + && (self.uuid.is_some() || self.stkg_sha256.is_some()) + && self.uuid.map_or(true, |known| uuid == Some(known)) + && self.stkg_sha256.map_or(true, |known| known == *digest) + } +} + +/// T6030 GPU firmware images known to the M3 runtime. An image is accepted when every recorded +/// identity (UUID, hash) matches. +pub(crate) static KNOWN_IMAGES: [KnownImage; 2] = [ + KnownImage { + name: "J514S RTKit-2419.140.12", + uuid: None, + stkg_sha256: Some(crate::m3_firmware::TEXT_SHA256), + initdata_magic: Some(G15_V14_8_3_INITDATA), + }, + KnownImage { + name: "g15s build b0 (firmware 14.8.3) RTKit-2419.140.12", + uuid: Some([ + 0xdb, 0xf3, 0x7c, 0x40, 0xea, 0xd5, 0x37, 0x60, 0x94, 0x41, 0x99, 0x50, + 0x18, 0x78, 0x29, 0x55, + ]), + stkg_sha256: None, + initdata_magic: Some(G15_V14_8_3_INITDATA), + }, +]; + +/// The G15 InitData version of firmware-compat 14.8.3: the version of both T6030 images and of +/// the T8122 image. +pub(crate) const G15_V14_8_3_INITDATA: u64 = 0x0c08_e21e_8380_0490; + +/// The image-info UUID of the T8122 C0 firmware-compat 14.8.3 image. +const T8122_C0_UUID: [u8; 16] = [ + 0xdf, 0x69, 0x7f, 0x05, 0xf6, 0xb5, 0x33, 0xef, 0xa1, 0x37, 0x61, 0xc4, 0xa7, 0x3d, 0x66, 0x6a, +]; + +/// T8122 C0, firmware-compat 14.8.3: the image the macOS 14.8.3 system firmware loads on the +/// M3 MacBook Airs. It takes the G15 14.8.3 InitData version. +pub(crate) static KNOWN_IMAGES_T8122: [KnownImage; 1] = [KnownImage { + name: "T8122 C0 firmware-compat 14.8.3", + uuid: Some(T8122_C0_UUID), + stkg_sha256: None, + initdata_magic: Some(G15_V14_8_3_INITDATA), +}]; + +/// The same T8122 image for the start experiment (`t8122_start`, only with +/// `asahi.t8122_start=1`), where `asahi.t8122_initdata_version` may replace the version. +pub(crate) static KNOWN_IMAGES_T8122_EXPERIMENT: [KnownImage; 1] = [KnownImage { + name: "T8122 C0 firmware-compat 14.8.3 (start experiment)", + uuid: Some(T8122_C0_UUID), + stkg_sha256: None, + initdata_magic: Some(G15_V14_8_3_INITDATA), +}]; + +const IMAGE_INFO_OFFSET: usize = 0x4200; +/// Size of the identifying part of the image-info header. +const IMAGE_INFO_SIZE: usize = 0x38; +/// First word of the header: a branch over it (`b +0x44`). +const IMAGE_INFO_BRANCH: u32 = 0x1400_0011; +/// Header magic, "uuid". +const IMAGE_INFO_MAGIC: u32 = 0x6469_7575; + +/// Parsed image-info header: UUID, patchbay offset and size, TEXT size. +pub(crate) struct ImageInfo { + pub(crate) uuid: [u8; 16], + pub(crate) patchbay: core::ops::Range, + pub(crate) text_size: usize, +} + +pub(crate) fn image_info(text: &[u8]) -> Option { + let hdr = text.get(IMAGE_INFO_OFFSET..IMAGE_INFO_OFFSET + IMAGE_INFO_SIZE)?; + let word = |i: usize| u32::from_le_bytes([hdr[i], hdr[i + 1], hdr[i + 2], hdr[i + 3]]); + if word(0) != IMAGE_INFO_BRANCH || word(4) != IMAGE_INFO_MAGIC || word(8) != 5 { + return None; + } + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&hdr[0x14..0x24]); + let start = word(0x2c) as usize; + let end = start.checked_add(word(0x30) as usize)?; + if word(0x34) as usize != text.len() { + return None; + } + text.get(start..end)?; + Some(ImageInfo { + uuid, + patchbay: start..end, + text_size: word(0x34) as usize, + }) +} + +/// The layout of a loaded GPU firmware the runtime accepts: segment sizes and VAs, and the +/// per-boot words of the text segment that identification ignores. +#[derive(Debug)] +pub(crate) struct Layout { + pub(crate) text_size: u64, + pub(crate) data_size: u64, + pub(crate) vas: [u64; 2], + /// Offset of the 8-byte tag that precedes the 8 per-boot bytes zeroed before hashing. + pub(crate) entropy_tag_offset: usize, + pub(crate) entropy_tag: [u8; 8], + /// The firmware version, for logs. + pub(crate) version: &'static str, +} + +pub(crate) static T6030_LAYOUT: Layout = Layout { + text_size: 0x5c000, + data_size: 0x114000, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0005_c000], + entropy_tag_offset: 0x58e88, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + version: "RTKit-2419.140.12.release", +}; + +/// T8122 firmware-compat 14.8.3 segment sizes, virtual addresses and entropy-tag bounds. +pub(crate) static T8122_LAYOUT: Layout = Layout { + text_size: 0x64000, + data_size: 0xb0000, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], + entropy_tag_offset: 0x5d54a, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + version: "RTKit-2419.140.12.release", +}; + +/// The T6031 firmware-compat 14.8.3 segment sizes and VAs, from the J516C ADT (iop-gfx-nub0 +/// segment-ranges: TEXT 0x64000 and DATA 0x140000 at the VAs both other layouts use). Neither +/// existing layout matches them. The boot-entropy tag offset is not known (0 stands for it); +/// an armed start searches the loaded text for the tag. The loaded image's version string is +/// not known ("unknown" stands for it). +pub(crate) const T6031_LAYOUT: Layout = Layout { + text_size: 0x64000, + data_size: 0x14_0000, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], + entropy_tag_offset: 0, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + version: "unknown", +}; + +pub(crate) const TEXT_SHA256: [u8; 32] = [ + 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; + +fn normalize_boot_entropy(layout: &Layout, text: &mut [u8]) -> bool { + let tag = layout.entropy_tag_offset; + if text.len() != layout.text_size as usize { + return false; + } + let Some(end) = tag.checked_add(16) else { return false; }; + let Some(entropy) = text.get_mut(tag..end) else { return false; }; + if entropy[..8] != layout.entropy_tag { + return false; + } + entropy[8..].fill(0); + true +} + +#[derive(Debug)] +pub(crate) struct Firmware { + pub(crate) resources: agx_resources::Resources, + /// The InitData version validated for the identified image. + pub(crate) initdata_magic: u64, + /// The accepted layout. + pub(crate) layout: &'static Layout, +} + +impl Firmware { + pub(crate) const fn version(&self) -> &'static str { + self.layout.version + } + + fn matching_layout(layout: &Layout, resources: &agx_resources::Resources) -> bool { + resources.regions[4].size == layout.text_size + && resources.regions[5].size == layout.data_size + && resources.firmware_vas == layout.vas + } +} + +#[cfg(not(test))] +pub(crate) fn identify_loaded( + pdev: &kernel::platform::Device, + soc: &'static crate::m3_soc::Soc, + resources: agx_resources::Resources, + experiment: Option<&crate::t8122_start::Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, +) -> kernel::error::Result { + use kernel::{ + bindings, c_str, + io::mem::{Mem, MemFlag}, + prelude::*, + }; + + let layout = soc.firmware.ok_or(ENODEV)?; + if !Firmware::matching_layout(layout, &resources) { + dev_err!(pdev.as_ref(), "M3 {}: unsupported firmware segment layout\n", soc.gpu_name); + return Err(ENODEV); + } + let node = pdev.as_ref().of_node().ok_or(ENODEV)?; + let text = agx_resources::reserved_resource(&node, c_str!("fw-text"))?; + if text.start() != resources.regions[4].base || text.size() != layout.text_size { + return Err(EINVAL); + } + let mapping = unsafe { Mem::try_new(text, MemFlag::WB.into()) }?; + let bytes = unsafe { core::slice::from_raw_parts(mapping.ptr(), mapping.size()) }; + let mut canonical = KVec::new(); + canonical.extend_from_slice(bytes, GFP_KERNEL)?; + // An armed T6031 searches the loaded text for its boot-entropy tag. The layout's offset is + // a placeholder, so the T8122 image table must not see this image. + if let Some(experiment) = t6031 { + experiment.normalize_text(&mut canonical)?; + let mut digest = [0u8; 32]; + // SAFETY: the initialized private copy and distinct digest live for the synchronous + // SHA-256 call. Normalization never writes the loaded firmware. + unsafe { bindings::sha256(canonical.as_ptr(), canonical.len(), digest.as_mut_ptr()) }; + experiment.admit_image(pdev.as_ref(), bytes, &digest)?; + return Ok(Firmware { + resources, + initdata_magic: experiment.initdata_version(), + layout, + }); + } + if !normalize_boot_entropy(layout, &mut canonical) { + return Err(ENODEV); + } + let mut digest = [0u8; 32]; + // SAFETY: the initialized private copy and distinct digest live for the + // synchronous SHA-256 call. Normalization never writes loaded firmware. + unsafe { bindings::sha256(canonical.as_ptr(), canonical.len(), digest.as_mut_ptr()) }; + // The T8122 start experiment identifies the same image, with its own InitData version. + let images = match experiment { + Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..], + None => soc.images, + }; + let image = crate::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; + let initdata_magic = match experiment { + Some(experiment) => experiment.initdata_version(), + None => image.initdata_magic.ok_or(ENODEV)?, + }; + Ok(Firmware { resources, initdata_magic, layout }) +} + +#[cfg(test)] +mod tests { + use super::*; + use agx_resources::{Region, Resources}; + + fn image_header(uuid: [u8; 16]) -> Vec { + let mut text = vec![0; T6030_LAYOUT.text_size as usize]; + let size = text.len() as u32; + for (offset, value) in [ + (0, IMAGE_INFO_BRANCH), (4, IMAGE_INFO_MAGIC), (8, 5), + (0x2c, size - 32), (0x30, 32), (0x34, size), + ] { + header_word(&mut text, offset, value); + } + text[IMAGE_INFO_OFFSET + 0x14..IMAGE_INFO_OFFSET + 0x24].copy_from_slice(&uuid); + text + } + + fn header_word(text: &mut [u8], offset: usize, value: u32) { + let start = IMAGE_INFO_OFFSET + offset; + text[start..start + 4].copy_from_slice(&value.to_le_bytes()); + } + + fn admits_text(image: &KnownImage, text: &[u8], digest: &[u8; 32]) -> bool { + image.accepts_runtime(image_info(text).map(|info| info.uuid), digest) + } + + #[test] + fn uuid_admission_requires_header_bounds_and_exact_text_size() { + let image = &KNOWN_IMAGES[1]; + let text = image_header(image.uuid.unwrap()); + assert!(admits_text(image, &text, &[0; 32])); + let size = text.len() as u32; + for (offset, value) in [ + (0x34, 0), (0x34, size - 1), (0x34, size + 1), + (0x2c, size), (0x2c, u32::MAX), (0x30, u32::MAX), + ] { + let mut bad = text.clone(); + header_word(&mut bad, offset, value); + assert!(!admits_text(image, &bad, &[0; 32]), "offset {offset:#x}, {value:#x}"); + } + let info = image_info(&text).unwrap(); + assert_eq!(info.patchbay, text.len() - 32..text.len()); + assert_eq!(info.text_size, text.len()); + } + + #[test] + fn uuid_admission_rejects_bad_header_words_and_truncation() { + let image = &KNOWN_IMAGES[1]; + let text = image_header(image.uuid.unwrap()); + for offset in [0, 4, 8] { + let mut bad = text.clone(); + header_word(&mut bad, offset, 0); + assert!(!admits_text(image, &bad, &[0; 32])); + } + for len in [0, IMAGE_INFO_OFFSET, IMAGE_INFO_OFFSET + IMAGE_INFO_SIZE - 1, text.len() - 1] { + assert!(!admits_text(image, &text[..len], &[0; 32])); + } + } + + #[test] + fn runtime_admission_requires_every_recorded_identity() { + let image = KnownImage { + name: "test", uuid: Some([1; 16]), stkg_sha256: Some([2; 32]), + initdata_magic: Some(3), + }; + let text = image_header([1; 16]); + assert!(admits_text(&image, &text, &[2; 32])); + assert!(!admits_text(&image, &text, &[0; 32])); + assert!(!admits_text(&image, &image_header([0; 16]), &[2; 32])); + assert!(!image.accepts_runtime(None, &[2; 32])); + let anonymous = KnownImage { uuid: None, stkg_sha256: None, ..image }; + assert!(!admits_text(&anonymous, &text, &[2; 32])); + } + + #[test] + fn digest_only_admission_preserves_exact_digest_requirement() { + let image = &KNOWN_IMAGES[0]; + assert!(admits_text(image, &[], &TEXT_SHA256)); + let mut wrong = TEXT_SHA256; + wrong[0] ^= 1; + assert!(!admits_text(image, &[], &wrong)); + } + + #[test] + fn t8122_admission_requires_qualified_initdata_and_matching_image_identity() { + let image = &KNOWN_IMAGES_T8122[0]; + assert_eq!(image.initdata_magic, Some(G15_V14_8_3_INITDATA)); + let text = image_header(image.uuid.unwrap()); + assert_eq!(image_info(&text).unwrap().uuid, image.uuid.unwrap()); + assert!(admits_text(image, &text, &[0; 32])); + + // A qualified version does not admit an unknown or another chip's image. + for wrong_uuid in [[0; 16], KNOWN_IMAGES[1].uuid.unwrap()] { + assert!(!admits_text(image, &image_header(wrong_uuid), &[0; 32])); + } + assert!(!image.accepts_runtime(None, &[0; 32])); + + // Matching identity still cannot admit an unqualified InitData layout. + let unqualified = KnownImage { + name: image.name, uuid: image.uuid, stkg_sha256: image.stkg_sha256, + initdata_magic: None, + }; + assert!(!admits_text(&unqualified, &text, &[0; 32])); + let unidentified = KnownImage { + name: "unknown", uuid: None, stkg_sha256: None, + initdata_magic: image.initdata_magic, + }; + assert!(!admits_text(&unidentified, &text, &[0; 32])); + } + + #[test] + fn t8122_experiment_record_identifies_the_same_image_with_the_g15_version() { + let image = &KNOWN_IMAGES_T8122_EXPERIMENT[0]; + assert_eq!(image.uuid, KNOWN_IMAGES_T8122[0].uuid); + assert_eq!(image.stkg_sha256, None); + assert_eq!(image.initdata_magic, Some(G15_V14_8_3_INITDATA)); + assert!(KNOWN_IMAGES.iter().all(|i| i.initdata_magic == Some(G15_V14_8_3_INITDATA))); + let text = image_header(image.uuid.unwrap()); + assert!(admits_text(image, &text, &[0; 32])); + assert!(!admits_text(image, &image_header(KNOWN_IMAGES[1].uuid.unwrap()), &[0; 32])); + assert!(!admits_text(&KNOWN_IMAGES[1], &text, &[0; 32])); + } + + #[test] + fn each_layout_rejects_the_other_chips_segments() { + for (layout, other) in [(&T6030_LAYOUT, &T8122_LAYOUT), (&T8122_LAYOUT, &T6030_LAYOUT)] { + let mut regions = [Region { base: 0, size: 0 }; 6]; + regions[4].size = layout.text_size; + regions[5].size = layout.data_size; + let mut resources = Resources { regions, firmware_vas: layout.vas }; + assert!(Firmware::matching_layout(layout, &resources)); + assert!(!Firmware::matching_layout(other, &resources)); + resources.firmware_vas[1] += 0x4000; + assert!(!Firmware::matching_layout(layout, &resources)); + } + } + + #[test] + fn t6031_layout_is_the_adt_segments_and_matches_neither_other_layout() { + let layout = &T6031_LAYOUT; + assert_eq!((layout.text_size, layout.data_size), (0x64000, 0x140000)); + assert_eq!(layout.vas, [0xffff_fc00_0000_0000, 0xffff_fc00_0000_0000 + 0x64000]); + assert_eq!(layout.entropy_tag, T8122_LAYOUT.entropy_tag); + let mut regions = [Region { base: 0, size: 0 }; 6]; + regions[4].size = layout.text_size; + regions[5].size = layout.data_size; + let resources = Resources { regions, firmware_vas: layout.vas }; + assert!(Firmware::matching_layout(layout, &resources)); + assert!(!Firmware::matching_layout(&T6030_LAYOUT, &resources)); + assert!(!Firmware::matching_layout(&T8122_LAYOUT, &resources)); + } + + #[test] + fn entropy_normalization_changes_only_the_eight_boot_bytes() { + for layout in [&T6030_LAYOUT, &T8122_LAYOUT] { + let mut text = vec![0xa5; layout.text_size as usize]; + let tag = layout.entropy_tag_offset; + text[tag..tag + 8].copy_from_slice(&layout.entropy_tag); + let before = text.clone(); + assert!(normalize_boot_entropy(layout, &mut text)); + assert_eq!(&text[..tag + 8], &before[..tag + 8]); + assert_eq!(&text[tag + 8..tag + 16], &[0; 8]); + assert_eq!(&text[tag + 16..], &before[tag + 16..]); + } + } + + #[test] + fn bad_entropy_bounds_tags_and_lengths_do_not_mutate_text() { + for offset in [8, 16, usize::MAX] { + let layout = Layout { text_size: 16, entropy_tag_offset: offset, ..T8122_LAYOUT }; + let mut text = vec![0xa5; 16]; + assert!(!normalize_boot_entropy(&layout, &mut text)); + assert_eq!(text, vec![0xa5; 16]); + } + let layout = Layout { text_size: 16, entropy_tag_offset: 0, ..T8122_LAYOUT }; + let mut text = vec![0xa5; 16]; + assert!(!normalize_boot_entropy(&layout, &mut text)); + assert_eq!(text, vec![0xa5; 16]); + text.pop(); + assert!(!normalize_boot_entropy(&layout, &mut text)); + assert_eq!(text, vec![0xa5; 15]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_memory.rs b/drivers/gpu/drm/asahi/m3/m3_memory.rs new file mode 100644 index 0000000000000..cbfb771fd0873 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_memory.rs @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned firmware/engine buffers with the qualified M3 page offsets. +use kernel::prelude::*; +use crate::{driver, gem, mmu, pgtable::prot}; + +enum Backing { + Coherent(kernel::dma::Coherent<[u8]>), + Paged(gem::ObjectRef), +} + +pub(crate) struct Buffer { + mapping: mmu::KernelMapping, + gpu_mapping: Option, + // Every client view must preserve the owner's cache and access policy. + // A VM switch changes the address space, not the backing's attributes. + gpu_protection: prot::Prot, + object: Backing, + offset: usize, + size: usize, + _coherent_allocation: Option, +} + +impl Buffer { + pub(crate) fn at(dev: &driver::AsahiDevice, fw: &mmu::Vm, + gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, + gpu_shared: bool) -> Result { + Self::at_prot(dev,fw,gpu,address,size, + if gpu_shared {prot::PROT_GPU_FW_SHARED_RW} else {prot::PROT_FW_SHARED_RW}, + prot::PROT_GPU_SHARED_RW) + } + pub(crate) fn at_prot(dev: &driver::AsahiDevice, fw: &mmu::Vm, + gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, + protection: prot::Prot, gpu_protection: prot::Prot) -> Result { + let offset=(address.or(gpu.map(|(_,a)|a)).unwrap_or(0)&0x3fff) as usize; + if size==0 || gpu.is_some_and(|(_,a)| a&0x3fff!=offset as u64) {return Err(EINVAL);} + let allocated=(size.checked_add(offset).and_then(|n|n.checked_add(0x3fff)).ok_or(EOVERFLOW)?)&!0x3fff; + // The DMA core tries the default CMA area first and falls back to the + // page allocator. The graphs outgrow that area, so a CMA miss is the + // normal case here and is not reported; only a failed fallback is. + // SAFETY: M3 runtime owns the bound platform device for the entire + // diagnostic; all mappings retire before normal DMA allocation Drop. + let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_bound()},allocated,GFP_KERNEL, + kernel::dma::attrs::DMA_ATTR_NO_WARN).inspect_err(|_| dev_err!(dev.as_ref(),"M3: cannot allocate {} KiB of GPU memory\n",allocated/1024))?; + let pa:usize=object.dma_handle().try_into()?; + let mapping=if let Some(a)=address {fw.map_io(a&!0x3fff,pa,allocated,protection)?} + else {fw.map_io_in_range(0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000,pa,allocated,protection)?}; + let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; + Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Coherent(object),offset,size, + _coherent_allocation:Some(crate::agx_memory_stats::Allocation::coherent(allocated)?)}) + } + /// Allocate owned tiler storage in the driver-reserved GPU range. Keep the + /// old allocation mapped until the replacement has been fully allocated. + pub(crate) fn tiler(dev:&driver::AsahiDevice,fw:&mmu::Vm,vm:&mmu::Vm,size:usize)->Result { + if size==0 || size.checked_add(mmu::UAT_PGMSK).is_none() {return Err(EINVAL);} + // Large render targets can require 128 MiB of tiler storage. Engines + // and firmware address it through UAT, so physically contiguous DMA + // backing is unnecessary and can exceed the entire CMA reservation. + // GEM provides zeroed pages and the same WC CPU/device coherency used + // by M4. Keep both mappings ahead of their backing in drop order. + let mut object=gem::new_kernel_object_wc(dev,size)?; + let mapping=object.map_into_range(fw,0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000, + mmu::UAT_PGSZ as u64,prot::PROT_GPU_FW_SHARED_RW,true)?; + // Growth retains the cached GPU-only policy of the original TPC and + // tilemap allocations; the firmware alias remains shared/uncached. + let gpu_protection=prot::Prot::from_pte(crate::m3_pass_layout::Access::GpuCachedRw.pte()); + let gpu_mapping=Some(object.map_into_range(vm,0x74_0000_0000..0x78_0000_0000, + mmu::UAT_PGSZ as u64,gpu_protection,true)?); + Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Paged(object),offset:0,size,_coherent_allocation:None}) + } + pub(crate) fn va(&self)->u64 {self.mapping.iova()+self.offset as u64} + pub(crate) fn gpu_va(&self)->Result {Ok(self.gpu_mapping.as_ref().ok_or(EINVAL)?.iova()+self.offset as u64)} + pub(crate) fn size(&self)->usize {self.size} + pub(crate) fn map_gpu_view(&mut self,vm:&mmu::Vm)->Result { + let address=self.gpu_va()?&!0x3fff; + match &mut self.object { + Backing::Coherent(object)=>vm.map_io(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), + Backing::Paged(object)=>object.map_at(vm,address,self.gpu_protection,true), + } + } + /// The backing retains its CPU mapping until Buffer is dropped. Callers + /// check the range and do not let this pointer escape the current access. + fn cpu_ptr(&mut self)->Result<*mut u8> { + match &mut self.object { + Backing::Coherent(object)=>Ok(object.as_mut_ptr().cast::()), + Backing::Paged(object)=>Ok(object.cached_vmap()?.as_mut_ptr()), + } + } + + pub(crate) fn swap_gpu_view(&mut self, mapping: mmu::KernelMapping)->Result { + self.gpu_mapping.replace(mapping).ok_or(EINVAL) + } + pub(crate) fn write(&mut self,offset:usize,data:&[u8])->Result { + if offset.checked_add(data.len()).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + unsafe {core::ptr::copy_nonoverlapping(data.as_ptr(),ptr.add(self.offset+offset),data.len())}; Ok(()) + } + /// Initialize the complete owned span through its retained CPU mapping. + /// The caller must own the storage or have retired all device consumers. + pub(crate) fn fill(&mut self, value: u8) -> Result { + self.fill_range(0,self.size,value) + } + /// Reset one unpublished slot without touching other queued device owners. + pub(crate) fn fill_range(&mut self,offset:usize,size:usize,value:u8)->Result { + if offset.checked_add(size).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr = self.cpu_ptr()?; + // SAFETY: allocation validates offset + size against its backing, and + // cpu_ptr retains that entire coherent/WC mapping for this access. + // Resolve the mapping once, as the G16 buffer clear path does, instead + // of repeating cached_vmap and memcpy for every 256-byte chunk. + unsafe { core::ptr::write_bytes(ptr.add(self.offset+offset), value, size) }; + Ok(()) + } + pub(crate) fn u32(&mut self,offset:usize,value:u32)->Result { + if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?; + if start&3!=0 {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + // SAFETY: aligned, bounded WC GEM storage retained by this owner. + unsafe {ptr.add(start).cast::().write_volatile(value.to_le())};Ok(()) + } + pub(crate) fn u64(&mut self,offset:usize,value:u64)->Result {self.write(offset,&value.to_le_bytes())} + /// Publish the unaligned performance-request word with bounded byte stores. + pub(crate) fn packed_request(&mut self,offset:usize,value:u32)->Result { + if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + // SAFETY: checked span in retained CPU storage; no aligned access is required. + unsafe {crate::m3_consistent_perf::store_packed(ptr.add(self.offset+offset),value)}; + Ok(()) + } + pub(crate) fn read(&mut self,offset:usize,data:&mut[u8])->Result { + if offset.checked_add(data.len()).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + // SAFETY: checked WC GEM span; firmware may change the shared bytes. + unsafe { + core::arch::asm!("dmb osh",options(nostack,preserves_flags)); + for (i,b) in data.iter_mut().enumerate() {*b=ptr.cast_const().add(self.offset+offset+i).read_volatile();} + } Ok(()) + } + pub(crate) fn read_u32(&mut self,offset:usize)->Result { + if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?; + if start&3!=0 {let mut b=[0;4];self.read(offset,&mut b)?;return Ok(u32::from_le_bytes(b));} + let ptr=self.cpu_ptr()?; + // SAFETY: aligned, checked atomic producer/consumer word in WC RAM. + let value=unsafe {ptr.cast_const().add(start).cast::().read_volatile()}; + unsafe {core::arch::asm!("dmb oshld",options(nostack,preserves_flags))}; + Ok(u32::from_le(value)) + } + pub(crate) fn read_u64(&mut self,offset:usize)->Result { + if offset.checked_add(8).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?; + // Packed firmware records also contain unaligned u64 fields. Retain + // their byte-wise access rather than issuing an unaligned word load. + if start&7!=0 {let mut b=[0;8];self.read(offset,&mut b)?;return Ok(u64::from_le_bytes(b));} + let ptr=self.cpu_ptr()?; + // SAFETY: retained page-aligned coherent/WC backing, checked aligned + // eight-byte span. One volatile word load avoids eight separate reads + // and byte-wise tearing of an aligned firmware timestamp/counter. + let value=unsafe {ptr.cast_const().add(start).cast::().read_volatile()}; + unsafe {core::arch::asm!("dmb oshld",options(nostack,preserves_flags))}; + Ok(u64::from_le(value)) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs new file mode 100644 index 0000000000000..d34e1f23840eb --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -0,0 +1,900 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Module parameters of the T6030 (M3) backends. +//! +//! The `module!` parameter support only parses plain integers. The parameters here are declared +//! with their own `kernel_param` entries (the same layout the `module!` macro emits), so they can +//! take symbolic values on the kernel command line, e.g. `asahi.m3_backend=manager`, and hex or +//! decimal masks. + +use core::ffi::{ + c_char, + c_int, +}; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +use kernel::{ + device::Core, + platform, + prelude::*, // +}; + +/// Parse a decimal or `0x`-prefixed hexadecimal `u64`. +fn parse_u64(text: &str) -> Option { + let text = text.trim(); + match text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")) { + Some(hex) => u64::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + } +} + +/// Store the parsed value of a parameter string into the `AtomicU64` behind `kp`. +/// +/// # Safety +/// +/// `val` must be NULL or a NUL-terminated string, and `kp` must be a `kernel_param` declared by +/// [`m3_param!`], whose `arg` points at a static `AtomicU64`. +unsafe fn set_param( + val: *const c_char, + kp: *const kernel::bindings::kernel_param, + parse: fn(&str) -> Option, +) -> c_int { + if val.is_null() { + return EINVAL.to_errno(); + } + // SAFETY: `val` is a NUL-terminated string per the function contract. + let Ok(text) = unsafe { core::ffi::CStr::from_ptr(val) }.to_str() else { + return EINVAL.to_errno(); + }; + let Some(value) = parse(text) else { + return EINVAL.to_errno(); + }; + // SAFETY: `arg` points at the static `AtomicU64` this entry was declared with. + let slot = unsafe { &*(*kp).__bindgen_anon_1.arg.cast::() }; + slot.store(value, Ordering::Relaxed); + 0 +} + +/// Declare a module parameter `asahi.` stored in the static `AtomicU64` `$storage` and +/// parsed by `$parse: fn(&str) -> Option`. Not visible in sysfs. +macro_rules! m3_param { + ($name:literal, $storage:ident, $parse:path) => { + m3_param!($name, $storage, $parse, 0, None); + }; + ($name:literal, $storage:ident, $parse:path, $perm:expr, $get:expr) => { + const _: () = { + unsafe extern "C" fn set( + val: *const c_char, + kp: *const kernel::bindings::kernel_param, + ) -> c_int { + // SAFETY: Called by the parameter parser with a value string (or NULL) and the + // `kernel_param` entry declared below. + unsafe { set_param(val, kp, $parse) } + } + + static OPS: kernel::bindings::kernel_param_ops = kernel::bindings::kernel_param_ops { + flags: 0, + set: Some(set), + get: $get, + free: None, + }; + + #[link_section = "__param"] + #[used(compiler)] + static PARAM: kernel::module_param::KernelParam = + kernel::module_param::KernelParam::new(kernel::bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + kernel::c_str!($name) + } else { + kernel::c_str!(concat!("asahi.", $name)) + }), + // `__this_module` is constructed by the kernel at load time and is not + // freed until the module is unloaded. Take its address without creating a + // reference to the mutable static. + #[cfg(MODULE)] + mod_: (&raw const kernel::bindings::__this_module).cast_mut(), + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&OPS), + perm: $perm, + level: -1, + flags: 0, + __bindgen_anon_1: kernel::bindings::kernel_param__bindgen_ty_1 { + arg: core::ptr::from_ref(&$storage).cast_mut().cast(), + }, + }); + }; + }; +} + +/// Read an atomic parameter without borrowing storage that a sysfs write may +/// change concurrently. The module-parameter core supplies a PAGE_SIZE buffer. +/// +/// # Safety +/// +/// `buffer` must be the module-parameter core's writable output buffer and +/// `kp` must be an entry declared by `m3_param!` with AtomicU64 storage. +unsafe extern "C" fn get_atomic_param( + buffer: *mut c_char, + kp: *const kernel::bindings::kernel_param, +) -> c_int { + use core::fmt::Write; + + // SAFETY: The entry points to static AtomicU64 storage by contract. Loads + // and stores are atomic; module-parameter serialization alone would not + // protect the scheduler's concurrent reads. + let value = unsafe { &*((*kp).__bindgen_anon_1.arg.cast::()) } + .load(Ordering::Relaxed); + // SAFETY: 32 bytes fit in the core's PAGE_SIZE output buffer and hold any + // decimal u64 plus its NUL. The core appends the sysfs newline itself. + let bytes = unsafe { core::slice::from_raw_parts_mut(buffer.cast::(), 32) }; + let mut output = kernel::str::Formatter::new(bytes); + if write!(output, "{}\0", value).is_err() { + return EINVAL.to_errno(); + } + (output.bytes_written() - 1) as c_int +} + +fn parse_render_batch_override(text: &str) -> Option { + parse_u64(text).filter(|value| *value <= crate::m3_pass_layout::SLOTS as u64) +} + +/// Runtime tuning is opt-in: zero keeps the existing boot parameter and its +/// clamping semantics. Only root may write a bounded 1..=SLOTS override; an +/// invalid write leaves the previous value intact. Compute batching is separate. +static M3_RENDER_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_render_batch_override", M3_RENDER_BATCH_OVERRIDE, + parse_render_batch_override, 0o644, Some(get_atomic_param)); + +fn parse_render_stamp_coalesce(text:&str)->Option { + parse_u64(text).filter(|value| *value<=1) +} + +/// Root-only T6030 render stamp-flush coalescing; zero retains per-pass flushes. +/// Each publication snapshots the setting before encoding any pass. +static M3_RENDER_STAMP_COALESCE: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_render_stamp_coalesce", M3_RENDER_STAMP_COALESCE, + parse_render_stamp_coalesce, 0o644, Some(get_atomic_param)); +pub(crate) fn render_stamp_coalesce()->bool { + M3_RENDER_STAMP_COALESCE.load(Ordering::Relaxed)!=0 +} + +/// T6030 cross-engine publications retain GPU ordering with firmware barriers. +/// Zero preserves the existing host retirement between engine groups. +static M3_MIXED_BATCH: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_mixed_batch", M3_MIXED_BATCH, + parse_render_stamp_coalesce, 0o644, Some(get_atomic_param)); +pub(crate) fn mixed_batch()->bool { + M3_MIXED_BATCH.load(Ordering::Relaxed)!=0 +} + +/// Boot-only T6030 power-state sampling while GPU stamps are outstanding. +/// Zero preserves the completion path without additional register reads. +static M3_ACTIVE_POWER: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_active_power", M3_ACTIVE_POWER, parse_render_stamp_coalesce); +pub(crate) fn active_power()->bool { + M3_ACTIVE_POWER.load(Ordering::Relaxed)==1 +} + +/// Boot-only T6030 Globals debug-word experiment. Zero preserves the runtime word. +static M3_GLOBALS_DEBUG_OFF: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_globals_debug_off",M3_GLOBALS_DEBUG_OFF,parse_render_stamp_coalesce); +pub(crate) fn globals_debug_off()->bool {M3_GLOBALS_DEBUG_OFF.load(Ordering::Relaxed)==1} + +/// T6030-only request while work is pending or active. Zero leaves firmware +/// performance selection unchanged; three requests the top published state. +static M3_CONSISTENT_PERF: AtomicU64 = AtomicU64::new(0); +fn parse_consistent_perf(text: &str)->Option { + parse_u64(text).filter(|v| matches!(*v,0|3)) +} +m3_param!("m3_consistent_perf",M3_CONSISTENT_PERF,parse_consistent_perf,0o644,Some(get_atomic_param)); +pub(crate) fn consistent_perf()->u64 {M3_CONSISTENT_PERF.load(Ordering::Relaxed)} + +/// One preserves internal render measurements. Application timestamp requests +/// remain recorded when zero omits measurements on T6030. +static M3_RENDER_INTERNAL_TIMESTAMPS: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_render_internal_timestamps", M3_RENDER_INTERNAL_TIMESTAMPS, + parse_render_stamp_coalesce, 0o644, Some(get_atomic_param)); +pub(crate) fn render_internal_timestamps()->bool { + M3_RENDER_INTERNAL_TIMESTAMPS.load(Ordering::Relaxed)!=0 +} + +/// Snapshot once per batch. A live change affects later publications, never the +/// storage, ordering or retirement checks of work already on the firmware queues. +pub(crate) fn render_batch_size() -> usize { + let override_size = M3_RENDER_BATCH_OVERRIDE.load(Ordering::Relaxed); + let size = if override_size == 0 { + *crate::module_parameters::m3_render_batch_size.value() as usize + } else { + override_size as usize + }; + size.clamp(1, crate::m3_pass_layout::SLOTS) +} + +fn parse_compute_batch_override(text: &str) -> Option { + parse_u64(text).filter(|value| *value <= crate::m3_compute_storage::SLOTS as u64) +} + +/// Independently tune the existing adjacent-compute path. Zero preserves the +/// boot parameter; invalid writes do not replace the current value. Storage and +/// engine/VM retirement boundaries are unchanged. +static M3_COMPUTE_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_compute_batch_override", M3_COMPUTE_BATCH_OVERRIDE, + parse_compute_batch_override, 0o644, Some(get_atomic_param)); + +/// Snapshot once per packet, so a live write never changes an in-flight batch. +pub(crate) fn compute_batch_size() -> usize { + let override_size = M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed); + let size = if override_size == 0 { + *crate::module_parameters::m3_compute_batch_size.value() as usize + } else { + override_size as usize + }; + size.clamp(1, crate::m3_compute_storage::SLOTS) +} + +/// `asahi.g15_debug`: G15 bring-up bits, see [`G15Debug`]. +static G15_DEBUG: AtomicU64 = AtomicU64::new(0); +m3_param!("g15_debug", G15_DEBUG, parse_u64); + +/// Bits of `asahi.g15_debug`, the bring-up switches of the G15 manager backend. +/// +/// The bit numbers are unchanged from their earlier `asahi.debug_flags` positions. Bits 49 +/// and 50 are no longer switches: the HwDataB power-management flags they enabled are always +/// set on G15. +#[derive(Copy, Clone)] +#[repr(u32)] +pub(crate) enum G15Debug { + /// Debug switch: issue the Fender kick (0x11, then 0x10) before starting the ASC. The + /// G15 firmware boots and runs jobs without it. + FenderKick = 41, + /// Register the DRM device (card/render nodes) for the manager backend, like + /// `asahi.m3_expose=1`. Off by default so compositors and system Mesa never see a GPU that + /// cannot run userspace jobs yet; the firmware bring-up runs either way. + ExposeDrm = 42, + /// Turn the firmware log on (RuntimePointers+0x230 = 1) before MSG_INIT. The firmware + /// writes no log records while it is 0. + FwLog = 43, + /// A/B: ring a DEVCTRL (0x83|0x11) and a KICKFW (0x83|0x10) doorbell immediately after + /// MSG_INIT, before the firmware has registered its EP 0x21 handler. + LegacyDoorbells = 44, + /// Skip the post-init DeviceControl liveness probe (no doorbell at all after MSG_INIT). + NoDevctlProbe = 45, + /// Map the gfx-data carveout (firmware data) read-only and log the init progress words, + /// thread handles and crashlog fill at every health sample. + GfxDataPeek = 46, + /// After firmware init, submit one empty compute job and poll its stamp for 2 seconds. + /// Never fails probe and never blocks boot past that poll. + SelfTest = 47, + /// With [`G15Debug::SelfTest`]: first run a firmware-only copy of the job (cmd+0x878 = 1, + /// StartCL skip path, no hardware kick). The real job runs only if that one completes. + SelfTestSkipFirst = 51, + /// Runtime backend: stop the probe right before the GPU coprocessor would be started. The + /// firmware never runs; everything before that point (admission, firmware identity, + /// InitData generation, register identity checks) still does. + StopBeforeAsc = 53, + /// A/B for the manager backend: publish the performance states as the two voltage-sorted + /// tables (with their device-tree index maps) instead of one table in device-tree order. + ManagerSplitPstates = 54, + /// A/B for the manager backend: write the power-controller block's PPM words at +0xc8..+0xd8 + /// in the order the runtime backend's InitData uses (0, enable, target power, kp, ki*dt), + /// and the Globals power-interface targets and performance-state cap, instead of the + /// manager's validated values. + ManagerReferencePpm = 55, + /// Debug switch for the runtime backend: when a job fails and the firmware reports that it has + /// halted, clear the halted flag and set resume, then log for about half a second what the + /// firmware, the engines and the failed job do. The GPU stays marked failed either way. + M3ResumeAfterFault = 56, +} + +/// Returns whether a `asahi.g15_debug` bit is set. +pub(crate) fn g15_debug(bit: G15Debug) -> bool { + G15_DEBUG.load(Ordering::Relaxed) & (1u64 << (bit as u32)) != 0 +} + +/// The raw `asahi.g15_debug` value, for logging. +pub(crate) fn g15_debug_mask() -> u64 { + G15_DEBUG.load(Ordering::Relaxed) +} + +/// `asahi.m3_backend` values. +const BACKEND_AUTO: u64 = 0; +const BACKEND_RUNTIME: u64 = 1; +const BACKEND_MANAGER: u64 = 2; +const BACKEND_OFF: u64 = 3; + +fn parse_backend(text: &str) -> Option { + match text.trim() { + "auto" | "0" => Some(BACKEND_AUTO), + "runtime" | "1" => Some(BACKEND_RUNTIME), + "manager" | "2" => Some(BACKEND_MANAGER), + "off" | "3" => Some(BACKEND_OFF), + _ => None, + } +} + +/// `asahi.m3_backend=auto|runtime|manager|off`: which T6030 GPU backend probe starts. +static M3_BACKEND: AtomicU64 = AtomicU64::new(BACKEND_AUTO); +m3_param!("m3_backend", M3_BACKEND, parse_backend); + +/// The T6030 GPU backend to start. +pub(crate) enum T6030Backend { + /// The serialized M3 runtime (m3_drm / m3_runtime). + Runtime, + /// The G15 GpuManager backend (g15_probe). + Manager, + /// No backend: the probe stops without touching the GPU. + Off, +} + +/// `asahi.m3_initdata` values. +const INITDATA_AUTO: u64 = 0; +const INITDATA_ADT: u64 = 2; + +fn parse_initdata(text: &str) -> Option { + match text.trim() { + "auto" | "0" => Some(INITDATA_AUTO), + "adt" | "2" => Some(INITDATA_ADT), + _ => None, + } +} + +/// `asahi.m3_initdata=auto|adt`: the runtime backend's InitData contents are generated from the +/// device tree either way; the value is logged. +static M3_INITDATA: AtomicU64 = AtomicU64::new(INITDATA_AUTO); +m3_param!("m3_initdata", M3_INITDATA, parse_initdata); + +/// Requested source of the runtime backend's InitData contents. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitDataParam { + /// Not given: contents generated from the device tree. + Auto, + /// Contents generated from the device tree. + Adt, +} + +/// The `asahi.m3_initdata` setting. +pub(crate) fn initdata_param() -> InitDataParam { + match M3_INITDATA.load(Ordering::Relaxed) { + INITDATA_ADT => InitDataParam::Adt, + _ => InitDataParam::Auto, + } +} + +/// Default of `asahi.m3_max_pstate` for device-tree InitData and the G15 manager backend: the +/// fifth performance state (1056 MHz on J516S). +pub(crate) const MAX_PSTATE_DEFAULT: u64 = 5; +/// A performance-state parameter that is not given. +const PSTATE_UNSET: u64 = u64::MAX; + +/// `asahi.m3_max_pstate`: the highest GPU performance state the driver lets the firmware use, +/// as an index into the voltage-sorted performance-state table (1 = lowest, 8 = highest on +/// T6030; the G15 manager backend translates it to its device-tree ordered table). It is +/// clamped to the device tree's highest performance state. Unset means +/// [`MAX_PSTATE_DEFAULT`], except with the runtime backend while its thermal limit is on or +/// holding, which keeps the whole table. +static M3_MAX_PSTATE: AtomicU64 = AtomicU64::new(PSTATE_UNSET); +m3_param!("m3_max_pstate", M3_MAX_PSTATE, parse_u64); + +/// `asahi.m3_boot_pstate`: the performance state the runtime backend requests at boot, clamped +/// to the cap. Unset means the cap. +static M3_BOOT_PSTATE: AtomicU64 = AtomicU64::new(PSTATE_UNSET); +m3_param!("m3_boot_pstate", M3_BOOT_PSTATE, parse_u64); + +/// The `asahi.m3_max_pstate` setting, if given, before clamping to the performance-state table. +pub(crate) fn max_pstate_param() -> Option { + Some(M3_MAX_PSTATE.load(Ordering::Relaxed)).filter(|v| *v != PSTATE_UNSET) +} + +/// The `asahi.m3_boot_pstate` setting, if given, before clamping. +pub(crate) fn boot_pstate_param() -> Option { + Some(M3_BOOT_PSTATE.load(Ordering::Relaxed)).filter(|v| *v != PSTATE_UNSET) +} + +/// `asahi.m3_timeout_nohang`: 1 (default) makes a scheduler timeout of a runtime-backend job +/// report "no hang", because the runtime completes every packet itself and enforces its own +/// per-batch completion limit. 0 keeps the previous handling: mark the GPU failed, fail the +/// packet again and report the device as gone. +static M3_TIMEOUT_NOHANG: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_timeout_nohang", M3_TIMEOUT_NOHANG, parse_u64); + +/// Whether `asahi.m3_timeout_nohang` is set (the default). +pub(crate) fn timeout_nohang() -> bool { + M3_TIMEOUT_NOHANG.load(Ordering::Relaxed) != 0 +} + +/// `asahi.m3_unlocked_wait`: 1 (default) releases the runtime lock while the runtime backend +/// waits for a batch to complete, so VM creation, timestamp mapping and the event worker are +/// not held up by a running job. 0 holds the lock from preparation to retirement, as before. +static M3_UNLOCKED_WAIT: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_unlocked_wait", M3_UNLOCKED_WAIT, parse_u64); + +/// Whether `asahi.m3_unlocked_wait` is set (the default). +pub(crate) fn unlocked_wait() -> bool { + M3_UNLOCKED_WAIT.load(Ordering::Relaxed) != 0 +} + +/// `asahi.m3_ut_engagement=0|1`: Globals +0x7d0, the gate of the firmware's frequency-feedback +/// cap. Unset: the SoC's default ([`crate::m3_soc::Soc::ut_engagement`]). +static M3_UT_ENGAGEMENT: AtomicU64 = AtomicU64::new(UT_ENGAGEMENT_UNSET); +const UT_ENGAGEMENT_UNSET: u64 = u64::MAX; +fn parse_ut_engagement(text: &str) -> Option { + parse_u64(text).filter(|v| *v <= 1) +} +m3_param!("m3_ut_engagement", M3_UT_ENGAGEMENT, parse_ut_engagement); + +/// The frequency-feedback gate for `soc`, and whether `asahi.m3_ut_engagement` set it. +pub(crate) fn ut_engagement(soc: &crate::m3_soc::Soc) -> (u32, bool) { + match M3_UT_ENGAGEMENT.load(Ordering::Relaxed) { + UT_ENGAGEMENT_UNSET => (soc.ut_engagement, false), + v => (v as u32, true), + } +} + +/// `asahi.m3_pipeline_depth`: writable at runtime. How many render batches may be in flight on +/// the firmware's render queues. Zero selects the SoC default: 2 on the explicitly enabled +/// experimental T8122 runtime, 1 on T6030. Depth 1 publishes only after the previous batch has +/// fully retired. Larger values publish the next render batch of the same VM while earlier +/// ones still run, in disjoint pass slots, ordered by the existing cross-pass barriers. +static M3_PIPELINE_DEPTH: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_pipeline_depth", M3_PIPELINE_DEPTH, parse_pipeline_depth, 0o644, Some(get_atomic_param)); + +fn parse_pipeline_depth(text: &str) -> Option { + parse_u64(text).filter(|v| *v<=crate::m3_pass_layout::SLOTS as u64) +} + +/// Effective depth; changing it never releases already published pass slots. +pub(crate) fn pipeline_depth(soc: &crate::m3_soc::Soc) -> usize { + match M3_PIPELINE_DEPTH.load(Ordering::Relaxed) { + 0 => crate::m3_runtime::policy::default_depth(crate::t8122_start::is_t8122(soc)), + v => v.clamp(1,crate::m3_pass_layout::SLOTS as u64) as usize, + } +} + +/// `asahi.m3_render_batch_budget_us`: writable at runtime. A render batch takes in later packets +/// of its VM only while the VM's measured GPU time per pass, times the passes already in the +/// batch, stays below this budget (4000 µs on experimental T8122, disabled on Pro). +/// Coalescing a client's heavy frames +/// otherwise makes every other client's work, the compositor's included, wait for all of them. +/// 0 disables the budget; u64::MAX restores the SoC default. +static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX); +m3_param!("m3_render_batch_budget_us", M3_RENDER_BATCH_BUDGET_US, parse_u64, 0o644, Some(get_atomic_param)); + +/// The render batch GPU-time budget in ns, or `None` without one. +pub(crate) fn render_batch_budget_ns(soc: &crate::m3_soc::Soc) -> Option { + let value=M3_RENDER_BATCH_BUDGET_US.load(Ordering::Relaxed); + let us=if value==u64::MAX {crate::m3_runtime::policy::default_budget_us(crate::t8122_start::is_t8122(soc))} else {value}; + match us { + 0 => None, + us => Some(us.saturating_mul(1000)), + } +} + +/// Bits of `asahi.m3_retire_mmio`. +pub(crate) const RETIRE_MMIO_BUSY: u64 = 1 << 0; +pub(crate) const RETIRE_MMIO_FAULTS: u64 = 1 << 1; +pub(crate) const RETIRE_MMIO_PSTATE: u64 = 1 << 2; +const RETIRE_MMIO_UNSET: u64 = u64::MAX; + +fn parse_retire_mmio(text: &str) -> Option { + parse_u64(text).filter(|v| *v <= 7) +} + +/// `asahi.m3_retire_mmio`, writable at runtime: the GPU registers the host reads after every +/// job before retiring it. Bit 0 reads the engine-busy registers (SGX+0xc020/0xc120), bit 1 +/// selects and reads each cluster's fault bank (0xd800/0xd8c0), bit 2 reads the performance +/// state (0xe01000). Unset: the SoC's default ([`crate::m3_soc::Soc::retire_mmio`]). +static M3_RETIRE_MMIO: AtomicU64 = AtomicU64::new(RETIRE_MMIO_UNSET); +m3_param!("m3_retire_mmio", M3_RETIRE_MMIO, parse_retire_mmio, 0o644, Some(get_atomic_param)); + +/// The `asahi.m3_retire_mmio` mask for `soc`. +pub(crate) fn retire_mmio(soc: &crate::m3_soc::Soc) -> u64 { + match M3_RETIRE_MMIO.load(Ordering::Relaxed) { + RETIRE_MMIO_UNSET => soc.retire_mmio, + mask => mask, + } +} + +/// `asahi.m3_thermal` values. +const THERMAL_OFF: u64 = 0; +const THERMAL_HOLD: u64 = 1; +const THERMAL_ON: u64 = 2; +const THERMAL_UNSET: u64 = u64::MAX; + +fn parse_thermal(text: &str) -> Option { + match text.trim() { + "off" | "0" => Some(THERMAL_OFF), + "hold" => Some(THERMAL_HOLD), + "on" | "1" => Some(THERMAL_ON), + _ => None, + } +} + +static M3_THERMAL: AtomicU64 = AtomicU64::new(THERMAL_UNSET); +m3_param!("m3_thermal", M3_THERMAL, parse_thermal); + +/// How the runtime backend limits the GPU performance state by temperature. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub(crate) enum ThermalMode { + /// Today's fixed cap: the firmware is given the cap as its highest state, and nothing + /// changes it at runtime. + Off, + /// The firmware is given the whole table as its highest state, and the runtime cap words + /// hold the fixed cap for the whole boot. The same performance as `Off`, if the firmware + /// follows the runtime cap: this checks that it does. + Hold, + /// As `Hold`, and the runtime cap follows the SoC die temperature, up to the whole table + /// while the die is cool. + On, +} + +/// The `asahi.m3_thermal` setting, if given. +pub(crate) fn thermal_param() -> Option { + match M3_THERMAL.load(Ordering::Relaxed) { + THERMAL_OFF => Some(ThermalMode::Off), + THERMAL_HOLD => Some(ThermalMode::Hold), + THERMAL_ON => Some(ThermalMode::On), + _ => None, + } +} + +/// Default and bounds of `asahi.m3_thermal_hot`, in degrees Celsius. +pub(crate) const THERMAL_HOT_DEFAULT: u64 = 80; +pub(crate) const THERMAL_HOT_MIN: u64 = 40; +pub(crate) const THERMAL_HOT_MAX: u64 = 85; + +/// `asahi.m3_thermal_hot`: the SoC die temperature (degrees Celsius) at or above which the thermal +/// limit lowers the runtime cap, clamped to [`THERMAL_HOT_MIN`]..=[`THERMAL_HOT_MAX`]. +static M3_THERMAL_HOT: AtomicU64 = AtomicU64::new(THERMAL_HOT_DEFAULT); +m3_param!("m3_thermal_hot", M3_THERMAL_HOT, parse_u64); + +/// The `asahi.m3_thermal_hot` setting and whether it was clamped. +pub(crate) fn thermal_hot_param() -> (u64, bool) { + let value = M3_THERMAL_HOT.load(Ordering::Relaxed); + let clamped = value.clamp(THERMAL_HOT_MIN, THERMAL_HOT_MAX); + (clamped, clamped != value) +} + +/// Whether to start the runtime backend on a T8122 (`asahi.m3_backend`), and say why not. +/// +/// The runtime starts only on the boards the T8122 table allows (the M3 MacBook Airs), with +/// `auto` and `runtime` alike: it has not run on any T8122, so nothing forces it onto another +/// board. The G15 manager backend is T6030-only. +pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { + let dev = pdev.as_ref(); + match M3_BACKEND.load(Ordering::Relaxed) { + BACKEND_OFF => { + dev_info!(dev, "M3: asahi.m3_backend=off: no GPU backend started\n"); + false + } + BACKEND_MANAGER => { + dev_info!( + dev, + "M3: asahi.m3_backend=manager: the G15 manager backend is T6030-only; no GPU backend started\n" + ); + false + } + _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T8122) => true, + _ => { + dev_info!( + dev, + "M3: no GPU backend on this T8122 board: the runtime backend may start only on the M3 MacBook Airs (J613, J615)\n" + ); + false + } + } +} + +/// Select the T6030 GPU backend from `asahi.m3_backend`, and say which one and why. +/// +/// `auto` starts the runtime backend on the boards it is validated on (the M3 Pro MacBook +/// Pros). On other T6030 boards it starts nothing until the runtime is validated there; +/// `asahi.m3_backend=runtime` starts it anyway. +pub(crate) fn t6030_backend(pdev: &platform::Device) -> T6030Backend { + let dev = pdev.as_ref(); + match M3_BACKEND.load(Ordering::Relaxed) { + BACKEND_RUNTIME => { + dev_info!(dev, "M3: asahi.m3_backend=runtime\n"); + T6030Backend::Runtime + } + BACKEND_MANAGER => { + dev_info!(dev, "M3: asahi.m3_backend=manager: starting the G15 manager backend\n"); + T6030Backend::Manager + } + BACKEND_OFF => { + dev_info!(dev, "M3: asahi.m3_backend=off: no GPU backend started\n"); + T6030Backend::Off + } + _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T6030) => T6030Backend::Runtime, + _ => { + dev_info!( + dev, + "M3: asahi.m3_backend=auto: no GPU backend on this board: the runtime backend is not validated here yet (asahi.m3_backend=runtime starts it, asahi.m3_backend=manager starts the G15 manager backend)\n" + ); + T6030Backend::Off + } + } +} + +fn parse_t8122_number(text: &str) -> Option { + Some(crate::t8122_knobs::parse_number(text)) +} + +fn parse_t8122_fender(text: &str) -> Option { + Some(crate::t8122_knobs::parse_fender(text)) +} + +fn parse_t8122_clkgen(text: &str) -> Option { + Some(crate::t8122_knobs::parse_clkgen(text)) +} + +fn parse_t8122_sgx_setup(text: &str) -> Option { + Some(crate::t8122_knobs::parse_sgx_setup(text)) +} + +// The T8122 start experiment (`t8122_start`). Each parameter is stored as given, unset or with a +// value its parser does not accept (`t8122_knobs`), so that the experiment can refuse rather +// than fall back to a default. +use crate::t8122_knobs::UNSET as T8122_UNSET; + +/// `asahi.t8122_start=1`: start the M3 runtime on a T8122 with the start experiment's values. +/// Only with a boot loader that handed the GPU over. +static T8122_START: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_start", T8122_START, parse_t8122_number); +/// `asahi.t8122_initdata_version=`: the InitData version given to the T8122 firmware. +static T8122_INITDATA_VERSION: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_initdata_version", T8122_INITDATA_VERSION, parse_t8122_number); +/// `asahi.t8122_fender=0x104000|0x12c000` (`rule`, `adt`): the Fender window size. +static T8122_FENDER: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_fender", T8122_FENDER, parse_t8122_fender); +/// `asahi.t8122_clkgen=e1c|e5c|none`: the GPU clock-generator IO mapping. +static T8122_CLKGEN: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_clkgen", T8122_CLKGEN, parse_t8122_clkgen); +/// `asahi.t8122_sgx_setup=none|t6030`: the SGX write made before the firmware starts. +static T8122_SGX_SETUP: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_sgx_setup", T8122_SGX_SETUP, parse_t8122_sgx_setup); +/// `asahi.t8122_unit_mask_a=`: HwDataB unit mask A (+0x17c0). +static T8122_UNIT_MASK_A: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_unit_mask_a", T8122_UNIT_MASK_A, parse_t8122_number); +/// `asahi.t8122_unit_mask_b=`: HwDataB unit mask B (+0x17c8). +static T8122_UNIT_MASK_B: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_unit_mask_b", T8122_UNIT_MASK_B, parse_t8122_number); +/// `asahi.t8122_pstate_cap=N`: the highest performance state the T8122 firmware may use. +static T8122_PSTATE_CAP: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_pstate_cap", T8122_PSTATE_CAP, parse_t8122_number); + +/// `asahi.t8122_hwdata_418c=0|1`: on a T8122 with the G15 14.8.3 firmware, 1 (the default) +/// writes the 20 words at HwDataA +0x418c ([`crate::t8122_hwdata`]); 0 leaves the +/// range zero, for an A/B test. Readable in sysfs. +static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::t8122_hwdata::DEFAULT); +m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::t8122_hwdata::parse, 0o444, + Some(get_atomic_param)); + +/// Whether `asahi.t8122_hwdata_418c` asks for the HwDataA +0x418c words (the default). +pub(crate) fn t8122_hwdata_418c() -> bool { + crate::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) +} + +/// The T8122 start-experiment parameters of this boot, as given. +pub(crate) fn t8122_params() -> crate::t8122_knobs::Raw { + let get = |p: &AtomicU64| p.load(Ordering::Relaxed); + crate::t8122_knobs::Raw { + start: get(&T8122_START), + initdata_version: get(&T8122_INITDATA_VERSION), + fender: get(&T8122_FENDER), + clkgen: get(&T8122_CLKGEN), + sgx_setup: get(&T8122_SGX_SETUP), + unit_mask_a: get(&T8122_UNIT_MASK_A), + unit_mask_b: get(&T8122_UNIT_MASK_B), + pstate_cap: get(&T8122_PSTATE_CAP), + } +} + +fn parse_t6031_number(text: &str) -> Option { + Some(crate::t6031_knobs::parse_number(text)) +} + +fn parse_t6031_fender(text: &str) -> Option { + Some(crate::t6031_knobs::parse_fender(text)) +} + +fn parse_t6031_clkgen(text: &str) -> Option { + Some(crate::t6031_knobs::parse_clkgen(text)) +} + +fn parse_t6031_sgx_setup(text: &str) -> Option { + Some(crate::t6031_knobs::parse_sgx_setup(text)) +} + +fn parse_t6031_mcache(text: &str) -> Option { + Some(crate::t6031_knobs::parse_mcache(text)) +} + +fn parse_t6031_aic_swint(text: &str) -> Option { + Some(crate::t6031_knobs::parse_aic_swint(text)) +} + +fn parse_t6031_slot7(text: &str) -> Option { + Some(crate::t6031_knobs::parse_slot7(text)) +} + +fn parse_t6031_gifaf(text: &str) -> Option { + Some(crate::t6031_knobs::parse_gifaf(text)) +} + +fn parse_t6031_slot(text: &str) -> Option { + Some(crate::t6031_knobs::parse_slot(text)) +} + +fn parse_t6031_ane(text: &str) -> Option { + Some(crate::t6031_knobs::parse_ane(text)) +} + +fn parse_t6031_mtr(text: &str) -> Option { + Some(crate::t6031_knobs::parse_mtr(text)) +} + +fn parse_t6031_fw_words(text: &str) -> Option { + Some(crate::t6031_knobs::parse_fw_words(text)) +} + +fn parse_t6031_hwdata_object(text: &str) -> Option { + Some(crate::t6031_knobs::parse_hwdata_object(text)) +} + +fn parse_t6031_rev_id(text: &str) -> Option { + Some(crate::t6031_knobs::parse_rev_id(text)) +} + +fn parse_t6031_tristate(text: &str) -> Option { + Some(crate::t6031_knobs::parse_tristate(text)) +} + +fn parse_t6031_tag(text: &str) -> Option { + Some(crate::t6031_knobs::parse_tag(text)) +} + +// The T6031 start (`t6031_start`). Each parameter is stored as given, unset or with a value its +// parser does not accept (`t6031_knobs`), so the start can refuse rather than fall back. +use crate::t6031_knobs::UNSET as T6031_UNSET; + +/// `asahi.t6031_start=1`: one attempt to start the M3 runtime on a T6031. Without it the probe +/// is the same refusal as before. +static T6031_START: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_start", T6031_START, parse_t6031_number); +/// `asahi.t6031_image_hash`: the first 8 bytes of the loaded text SHA-256, big-endian. +static T6031_IMAGE_HASH: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_image_hash", T6031_IMAGE_HASH, parse_t6031_number); +/// `asahi.t6031_tag=search` or a text offset of the boot-entropy tag. +static T6031_TAG: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_tag", T6031_TAG, parse_t6031_tag); +/// `asahi.t6031_initdata_version=`: the InitData version given to the T6031 firmware. +static T6031_INITDATA_VERSION: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_initdata_version", T6031_INITDATA_VERSION, parse_t6031_number); +/// `asahi.t6031_pstate_cap=N`: the highest performance state the T6031 firmware may use. +static T6031_PSTATE_CAP: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_pstate_cap", T6031_PSTATE_CAP, parse_t6031_number); +/// `asahi.t6031_fender=rule|adt`: the Fender window size. +static T6031_FENDER: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_fender", T6031_FENDER, parse_t6031_fender); +/// `asahi.t6031_clkgen=e5c|e1c|none`: the GPU clock-generator IO mapping. +static T6031_CLKGEN: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_clkgen", T6031_CLKGEN, parse_t6031_clkgen); +/// `asahi.t6031_sgx_setup=t6030|none`: the SGX write made before the firmware starts. +static T6031_SGX_SETUP: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_sgx_setup", T6031_SGX_SETUP, parse_t6031_sgx_setup); +/// `asahi.t6031_mcache=none|x2|x4|x8`. +static T6031_MCACHE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mcache", T6031_MCACHE, parse_t6031_mcache); +/// `asahi.t6031_aic_swint=page|reg`. +static T6031_AIC_SWINT: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_aic_swint", T6031_AIC_SWINT, parse_t6031_aic_swint); +/// `asahi.t6031_slot7=none|rule`. +static T6031_SLOT7: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot7", T6031_SLOT7, parse_t6031_slot7); +/// `asahi.t6031_gifaf=none|fender`. +static T6031_GIFAF: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_gifaf", T6031_GIFAF, parse_t6031_gifaf); +/// `asahi.t6031_slot21=none` or a physical address. +static T6031_SLOT21: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot21", T6031_SLOT21, parse_t6031_slot); +/// `asahi.t6031_slot24=none` or a physical address. +static T6031_SLOT24: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot24", T6031_SLOT24, parse_t6031_slot); +/// `asahi.t6031_slot28=none` or a physical address. +static T6031_SLOT28: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot28", T6031_SLOT28, parse_t6031_slot); +/// `asahi.t6031_ane=none|t6030|t8122` or a physical address. +static T6031_ANE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_ane", T6031_ANE, parse_t6031_ane); +/// `asahi.t6031_io_drop`: a mask of default slots to leave out. +static T6031_IO_DROP: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_io_drop", T6031_IO_DROP, parse_t6031_number); +/// `asahi.t6031_mtr=t6030|t8122|adt|none`. +static T6031_MTR: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mtr", T6031_MTR, parse_t6031_mtr); +/// `asahi.t6031_mtr_fast_die`: overrides the fast-die MTR mask. +static T6031_MTR_FAST_DIE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mtr_fast_die", T6031_MTR_FAST_DIE, parse_t6031_number); +/// `asahi.t6031_mtr_alarm`: overrides the alarm MTR mask. +static T6031_MTR_ALARM: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mtr_alarm", T6031_MTR_ALARM, parse_t6031_number); +/// `asahi.t6031_unit_mask_a=`: HwDataB unit mask A (+0x17c0). +static T6031_UNIT_MASK_A: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_unit_mask_a", T6031_UNIT_MASK_A, parse_t6031_number); +/// `asahi.t6031_unit_mask_b=`: HwDataB unit mask B (+0x17c8). +static T6031_UNIT_MASK_B: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_unit_mask_b", T6031_UNIT_MASK_B, parse_t6031_number); +/// `asahi.t6031_hwb_454`: HwDataB +0xa2c. +static T6031_HWB_454: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_454", T6031_HWB_454, parse_t6031_number); +/// `asahi.t6031_hwb_b20`: HwDataB +0xb20, the core slots. +static T6031_HWB_B20: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_b20", T6031_HWB_B20, parse_t6031_number); +/// `asahi.t6031_hwb_17b8`: HwDataB +0x17b8. +static T6031_HWB_17B8: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_17b8", T6031_HWB_17B8, parse_t6031_number); +/// `asahi.t6031_hwb_1818`: HwDataB +0x1818. +static T6031_HWB_1818: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_1818", T6031_HWB_1818, parse_t6031_number); +/// `asahi.t6031_fw_words=none|t8122`. +static T6031_FW_WORDS: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_fw_words", T6031_FW_WORDS, parse_t6031_fw_words); +/// `asahi.t6031_hwdata_object=aligned|fixed`. +static T6031_HWDATA_OBJECT: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwdata_object", T6031_HWDATA_OBJECT, parse_t6031_hwdata_object); +/// `asahi.t6031_gpu_core`: must be the table's G15C core type, 24. +static T6031_GPU_CORE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_gpu_core", T6031_GPU_CORE, parse_t6031_number); +/// `asahi.t6031_rev_id=auto` or 1 to 6. +static T6031_REV_ID: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_rev_id", T6031_REV_ID, parse_t6031_rev_id); +/// `asahi.t6031_csafr=auto|on|off`. +static T6031_CSAFR: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_csafr", T6031_CSAFR, parse_t6031_tristate); +/// `asahi.t6031_power_cap_mw`: the power target while the boot loader's model is a stand-in. +static T6031_POWER_CAP_MW: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_power_cap_mw", T6031_POWER_CAP_MW, parse_t6031_number); + +/// The T6031 start parameters of this boot, as given. The image UUID has no module parameter. +pub(crate) fn t6031_params() -> crate::t6031_knobs::Raw { + let get = |p: &AtomicU64| p.load(Ordering::Relaxed); + crate::t6031_knobs::Raw { + start: get(&T6031_START), + image_uuid: crate::t6031_knobs::UuidParam::Unset, + image_hash: get(&T6031_IMAGE_HASH), + tag: get(&T6031_TAG), + initdata_version: get(&T6031_INITDATA_VERSION), + pstate_cap: get(&T6031_PSTATE_CAP), + fender: get(&T6031_FENDER), + clkgen: get(&T6031_CLKGEN), + sgx_setup: get(&T6031_SGX_SETUP), + mcache: get(&T6031_MCACHE), + aic_swint: get(&T6031_AIC_SWINT), + slot7: get(&T6031_SLOT7), + gifaf: get(&T6031_GIFAF), + slot21: get(&T6031_SLOT21), + slot24: get(&T6031_SLOT24), + slot28: get(&T6031_SLOT28), + ane: get(&T6031_ANE), + io_drop: get(&T6031_IO_DROP), + mtr: get(&T6031_MTR), + mtr_fast_die: get(&T6031_MTR_FAST_DIE), + mtr_alarm: get(&T6031_MTR_ALARM), + unit_mask_a: get(&T6031_UNIT_MASK_A), + unit_mask_b: get(&T6031_UNIT_MASK_B), + hwb_454: get(&T6031_HWB_454), + hwb_b20: get(&T6031_HWB_B20), + hwb_17b8: get(&T6031_HWB_17B8), + hwb_1818: get(&T6031_HWB_1818), + fw_words: get(&T6031_FW_WORDS), + hwdata_object: get(&T6031_HWDATA_OBJECT), + gpu_core: get(&T6031_GPU_CORE), + rev_id: get(&T6031_REV_ID), + csafr: get(&T6031_CSAFR), + power_cap_mw: get(&T6031_POWER_CAP_MW), + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs new file mode 100644 index 0000000000000..273d2701c3f14 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Private storage for one retired/unpublished G15 render pass. A Pass retains +//! all its buffers and mappings until Render retires the shared queues. +use kernel::prelude::*; +use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_pass_layout as layout}; +use layout::{Field,Space}; + +pub(crate) struct Pass {buffers:KVec} +impl Pass { + pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, + slot:usize,clusters:u32,aliases:&mut KVec)->Result { + let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?; + for field in layout::FIELDS { + let a=layout::board_allocation(slot,field,clusters).map_err(|_|EINVAL)?; + let mut b=match a.space { + Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, + a.size,prot::PROT_GPU_FW_SHARED_RW,prot::Prot::from_pte(a.access.pte()))?, + Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(), + a.gpu_alias.map(|address|(uat.kernel_lower_vm(),address)),Some(a.address), + a.size,prot::Prot::from_pte(a.access.pte()),prot::PROT_GPU_SHARED_RW)?, + }; + if a.gpu_alias.is_some() {aliases.push(b.map_gpu_view(vm)?,GFP_KERNEL)?;} + buffers.push(b,GFP_KERNEL)?; + } + Ok(Self{buffers}) + } + pub(crate) fn get(&self,field:Field)->&Buffer {&self.buffers[field as usize]} + pub(crate) fn get_mut(&mut self,field:Field)->&mut Buffer {&mut self.buffers[field as usize]} + /// Caller must prove retirement, and must not clear a prepared sibling. + /// Dependency commands are rewritten completely by the sync encoder. + pub(crate) fn reset(&mut self,geometry:&crate::agx_render::Geometry)->Result { + // Tiler arenas grow to retain the largest prior target. The firmware + // descriptors use the current geometry's strides and layer count, so + // only that prefix is addressable by this pass. Clear it on every use, + // including when growing again after a smaller pass; the unused tail + // may still contain retired data and is never assumed to be zero. + for field in &layout::FIELDS[..layout::RESET_COUNT] { + let bytes=match field { + Field::Tpc=>usize::try_from(geometry.tpc_bytes).map_err(|_|EOVERFLOW)?, + Field::Tilemap=>usize::try_from(geometry.tilemap_bytes).map_err(|_|EOVERFLOW)?, + _=>self.get(*field).size(), + }; + self.get_mut(*field).fill_range(0,bytes,0)?; + } + Ok(()) + } +} +/// Cached client views retain their resource identity across switches. A pass +/// field cannot accidentally name a shared allocator or another pass's field. +#[derive(Clone,Copy)] +pub(crate) enum ViewOwner {Shared(usize),Pass{slot:usize,field:Field}} diff --git a/drivers/gpu/drm/asahi/m3/m3_pool.rs b/drivers/gpu/drm/asahi/m3/m3_pool.rs new file mode 100644 index 0000000000000..979c1a37bb202 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_pool.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Construct a firmware-owned UMA allocator once from its graph's allocations. +//! GPU mappings and counters outlive this initialization and remain owned by +//! Render/Compute. Callers must never invoke this on a live or replayed pool. +use kernel::prelude::*; +use crate::{m3_memory::Buffer, m3_pool_layout as wire, m3_queue_layout::FirmwareVa}; +pub(crate) struct Owners { + pub(crate) backing: core::ops::Range, + pub(crate) pages: usize, + pub(crate) table: usize, + pub(crate) manager: usize, + pub(crate) counter: usize, + pub(crate) table_index: u8, +} +impl Owners { + pub(crate) fn initialize(&self, objects: &mut [Buffer]) -> Result { + let buffers = objects.get(self.backing.clone()).ok_or(EINVAL)?; + let mut backings = KVec::with_capacity(buffers.len(), GFP_KERNEL)?; + for buffer in buffers { + backings.push(wire::GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?, GFP_KERNEL)?; + } + let pages = objects.get(self.pages).ok_or(EINVAL)?; + let pages = wire::GpuRegion::new(pages.gpu_va()?, pages.size()).map_err(|_| EINVAL)?; + let table = objects.get(self.table).ok_or(EINVAL)?; + let table = wire::GpuRegion::new(table.gpu_va()?, table.size()).map_err(|_| EINVAL)?; + let counter = FirmwareVa::new(objects.get(self.counter).ok_or(EINVAL)?.va()).map_err(|_| EINVAL)?; + if objects.get(self.manager).ok_or(EINVAL)?.size() < wire::MANAGER_SIZE { return Err(EINVAL); } + let pool = wire::Pool::new(&backings, pages, table, counter, self.table_index).map_err(|_| EINVAL)?; + let mut manager = [0; wire::MANAGER_SIZE]; + pool.encode(&mut manager).map_err(|_| EINVAL)?; + // Unpopulated entries remain explicitly zero from the fresh DMA owner. + for page in 0..wire::INITIAL_PAGES { + objects[self.pages].u64(page * 8, pool.page(page).map_err(|_| EINVAL)?)?; + } + for entry in 0..backings.len() { + objects[self.table].write(entry * wire::POOL_ENTRY_SIZE, + &pool.table_entry(entry).map_err(|_| EINVAL)?)?; + } + objects[self.manager].write(0, &manager) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs new file mode 100644 index 0000000000000..c2c67f5c3aa5a --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -0,0 +1,696 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +use kernel::prelude::*; +use crate::m3_queue_layout as queue; +use crate::m3_sync_layout as sync; +use crate::m3_state_layout as state; +use crate::m3_scene_layout as scene; +use crate::{m3_pass::{Pass,ViewOwner},m3_pass_layout::{self as pass_layout,Field as P,Space}}; +use crate::{m3_fragment_command as fragment, m3_tiler_command as tiler, m3_render_sequence as sequence, m3_init_layout::Region}; +use crate::{m3_parameter_layout as parameter, m3_pool_layout::GpuRegion}; +use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_shared_layout as s,agx_uapi}; +use crate::m3_timeline as timeline; +use s::{PB_FIRST,PB_GROUPS,PB_BLOCKS_PER_GROUP,PB_BLOCK_SIZE}; +const PB_BLOCKS:u32=(PB_GROUPS*PB_BLOCKS_PER_GROUP) as u32; +pub(crate) const PB_PAGES:u32=PB_BLOCKS*4; +// Serial execution permits retaining each client's views of the owned arenas. +// Bound the cache to four VMs including the active one; cache entries retain +// their ASID lease and all aliases until eviction after exact retirement. +struct CachedViews { + aliases:KVec, + views:KVec<(ViewOwner,mmu::KernelMapping)>, + // Release mappings before returning their ASID lease (Rust field order). + binding:mmu::VmBind, +} +pub(crate) struct Render { + _aliases:KVec, objects:KVec, + passes:KVec, init_bm:Buffer, + // All active mappings above must drop before their ASID binding. + _binding:mmu::VmBind, + stats: Region, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, + last_progress_ns:i64, + draw:u64, heads:[u16;2], cached_views:KVec, + slot:usize, batch_count:usize, early_count:u32, + /// Measurement packets in the pass most recently appended to its queues. + stage_timestamps:[bool;2], + /// First pass slot of the current batch; its passes use base..base+batch_count mod SLOTS. + base:usize, + checkpoint:(u64,[u16;2],usize,usize,u32,usize), + /// GPU clusters of the board: tiler accelerators (TPC storage) and preemption buffers. + clusters:u32, + /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). + registers:crate::m3_compute_layout::RegisterSet, +} +// Queue allocations belong to Render, not to a pass. Keep their fixed arenas +// and firmware/shared mapping permissions from the qualified allocation map. +// Scratch is GPU-visible firmware storage; it is not the application's VM. +struct QueueObjects { + info: usize, state: usize, ring: usize, scratch: usize, + tiler: bool, uuid: u32, +} +use s::{GPU_CONTEXT,JOB_LIST,BUFFER_MANAGER,PARAMETER_PAGE_LIST, + PARAMETER_BLOCK_LIST,NOTIFIER,BLOCK_CONTROL,MANAGER_MISC}; +const MANAGER_SCENE_COUNT: u32 = 48; +const TA_POOL_COUNTER: usize = 11; +const UMA_POOLS: [crate::m3_pool::Owners; 2] = [ + crate::m3_pool::Owners { backing: 0..8, pages: 8, table: 9, manager: 10, counter: 11, table_index: 0 }, + crate::m3_pool::Owners { backing: 12..20, pages: 20, table: 21, manager: 22, counter: 23, table_index: 1 }, +]; +const FRAGMENT_POOL_COUNTER: usize = 23; +// Retained producer identifier for cross-pass dependencies, G15 / V14_8_3. +const DEPENDENCY_UUID: u32 = 250912; +const RENDER_QUEUES: [QueueObjects; 2] = [ + QueueObjects { info: s::TA_QUEUE, state: s::TA_POINTERS, ring: s::TA_RING, + scratch: s::TA_SCRATCH, tiler: true, uuid: 0x7a0000 }, + QueueObjects { info: s::FRAGMENT_QUEUE, state: s::FRAGMENT_POINTERS, ring: s::FRAGMENT_RING, + scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 }, +]; +impl Render { + pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, + registers:crate::m3_compute_layout::RegisterSet)->Result { + let binding=uat.bind(vm)?; + let mut objects=KVec::new();let mut aliases=KVec::new(); + for index in 0..s::COUNT { + let a=s::allocation(index).map_err(|_|EINVAL)?; + let protection=prot::Prot::from_pte(a.access.pte()); + let b=match a.space { + Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, + a.size,prot::PROT_GPU_FW_SHARED_RW,protection)?, + Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address), + a.size,protection,prot::PROT_GPU_SHARED_RW)?, + }; + objects.push(b,GFP_KERNEL)?; + } + // InitBM is shared, not part of a pass. Preserve its qualified mapping. + let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(0xfffffc200062bfe0), + sync::INIT_BM_SIZE,prot::PROT_FW_SHARED_RW,prot::PROT_GPU_SHARED_RW)?; + let mut passes=KVec::with_capacity(pass_layout::SLOTS,GFP_KERNEL)?; + for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,&mut aliases)?,GFP_KERNEL)?;} + let mut sequence_bytes=KVec::new(); + sequence_bytes.resize(sequence::STORAGE,0u8,GFP_KERNEL)?; + let mut tiler_bytes=KVec::new(); + tiler_bytes.resize(tiler::SIZE,0u8,GFP_KERNEL)?; + let mut fragment_bytes=KVec::new(); + fragment_bytes.resize(fragment::SIZE,0u8,GFP_KERNEL)?; + let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers}; + // Lists and the matching manager are built before InitBM publication. + job.initialize_parameter_buffer()?; + for pool in &UMA_POOLS { pool.initialize(&mut job.objects)?; } + job.initialize_state()?; + job.initialize_queues()?; + dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024)); + crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; + Ok(job) + } + fn initialize_parameter_buffer(&mut self) -> Result { + let region = |owner: usize| GpuRegion::new(self.objects[owner].gpu_va()?, self.objects[owner].size()) + .map_err(|_| EINVAL); + let pages = region(PARAMETER_PAGE_LIST)?; + let blocks = region(PARAMETER_BLOCK_LIST)?; + let control = queue::FirmwareVa::new(self.objects[BLOCK_CONTROL].va()).map_err(|_| EINVAL)?; + let counter = queue::FirmwareVa::new(self.objects[s::BM_COUNTER].va()).map_err(|_| EINVAL)?; + let manager = parameter::Manager::new(pages, blocks, control, counter, PB_BLOCKS).map_err(|_| EINVAL)?; + let mut bytes = [0; parameter::MANAGER_SIZE]; + manager.encode(&mut bytes).map_err(|_| EINVAL)?; + for group in 0..PB_GROUPS { + let buffer = &self.objects[PB_FIRST + group]; + let backing = parameter::Backing::new(GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?) + .map_err(|_| EINVAL)?; + for block in 0..PB_BLOCKS_PER_GROUP { + self.objects[PARAMETER_BLOCK_LIST].write((group * PB_BLOCKS_PER_GROUP + block) * 8, + &backing.block(block).map_err(|_| EINVAL)?)?; + } + for page in 0..PB_BLOCKS_PER_GROUP * parameter::PAGES_PER_BLOCK { + self.objects[PARAMETER_PAGE_LIST].u32((group * PB_BLOCKS_PER_GROUP * parameter::PAGES_PER_BLOCK + page) * 4, + backing.page(page).map_err(|_| EINVAL)?)?; + } + } + self.objects[BUFFER_MANAGER].write(0, &bytes) + } + fn initialize_state(&mut self) -> Result { + let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL); + let notification = state::Notifier { threshold: address(s::EVENT_COUNT)?, context: self._binding.slot() }; + let list_owner = address(JOB_LIST)?; + let mut context = [0; state::CONTEXT_SIZE]; + state::context(&mut context).map_err(|_| EINVAL)?; + self.objects[GPU_CONTEXT].write(0, &context)?; + let mut notifier = [0; state::NOTIFIER_SIZE]; + notification.encode(&mut notifier).map_err(|_| EINVAL)?; + self.objects[NOTIFIER].write(0, ¬ifier)?; + let mut list = [0; state::JOB_LIST_SIZE]; + state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; + self.objects[JOB_LIST].write(0, &list)?; + let mut manager = [0; state::MANAGER_STATE_SIZE]; + state::block_control(&mut manager, PB_BLOCKS, PB_BLOCKS).map_err(|_| EINVAL)?; + self.objects[BLOCK_CONTROL].write(0, &manager)?; + state::manager_misc(&mut manager).map_err(|_| EINVAL)?; + self.objects[MANAGER_MISC].write(0, &manager)?; + for (owner, value) in [(TA_POOL_COUNTER, 1), (FRAGMENT_POOL_COUNTER, 1), + (s::EVENT_COUNT, 2), (s::BM_COUNTER, 1), + (s::TA_STAMP, timeline::stamp(s::STAMP_TA - 256, 0)), (s::TA_FW_STAMP, timeline::stamp(s::STAMP_TA - 256, 0)), + (s::FRAGMENT_STAMP, timeline::stamp(s::STAMP_FRAGMENT - 256, 0)), (s::FRAGMENT_FW_STAMP, timeline::stamp(s::STAMP_FRAGMENT - 256, 0))] { + let mut bytes = [0; state::MANAGER_STATE_SIZE]; + let bytes = bytes.get_mut(..self.objects[owner].size()).ok_or(EINVAL)?; + state::counter(bytes, value).map_err(|_| EINVAL)?; + self.objects[owner].write(0, bytes)?; + } + Ok(()) + } + fn initialize_queues(&mut self) -> Result { + for owner in &RENDER_QUEUES { + let address = |index: usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_| EINVAL); + let info = queue::QueueInfo { + state: address(owner.state)?, ring: address(owner.ring)?, + job_list: address(JOB_LIST)?, gpu_scratch: address(owner.scratch)?, + context: address(GPU_CONTEXT)?, uuid: owner.uuid, + }; + let seed_addresses=if owner.tiler { + [self.init_bm.va(),self.passes[0].get(P::TilerCommand).va()] + }else{[self.passes[0].get(P::TilerToFragment).va(),self.passes[0].get(P::FragmentCommand).va()]}; + let seeds=[queue::FirmwareVa::new(seed_addresses[0]).map_err(|_|EINVAL)?, + queue::FirmwareVa::new(seed_addresses[1]).map_err(|_|EINVAL)?]; + let mut bytes = [0; queue::INFO_SIZE]; + info.encode(&mut bytes).map_err(|_| EINVAL)?; + self.objects[owner.info].write(0, &bytes)?; + let mut state = [0; queue::STATE_SIZE]; + queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 0 } + .encode(&mut state).map_err(|_| EINVAL)?; + self.objects[owner.state].write(0, &state)?; + for index in 0..queue::QUALIFIED_CAPACITY { + let entry = queue::ring_entry(index, queue::QUALIFIED_CAPACITY, &seeds).map_err(|_| EINVAL)?; + self.objects[owner.ring].write(index as usize * 8, &entry)?; + } + } + Ok(()) + } + fn write_barrier(&mut self, owner: P, command: sync::Barrier) -> Result { + let mut bytes = [0; sync::BARRIER_SIZE]; + command.encode(&mut bytes).map_err(|_| EINVAL)?; + self.passes[self.slot].get_mut(owner).write(0, &bytes) + } + // Construct synchronization from the actual owners after resetting a pass, + // before publishing its ring entries. InitBM is shared and dispatched only + // for draw one; rewriting its bytes after retirement preserves the old path. + fn prepare_sync(&mut self, early: bool, draw: u64, + predecessor:Option) -> Result { + let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL); + let manager = address(BUFFER_MANAGER)?; + let ta_stamp = address(s::TA_FW_STAMP)?; + let fs_stamp = address(s::FRAGMENT_FW_STAMP)?; + let mut init = [0; sync::INIT_BM_SIZE]; + sync::InitBufferManager { context: self._binding.slot(), slot: 0, + block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) } + .encode(&mut init).map_err(|_| EINVAL)?; + self.init_bm.write(0, &init)?; + let ta_value = timeline::stamp(s::STAMP_TA - 256, draw); + let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw); + let previous=predecessor.unwrap_or(sync::CompletionDependency { + stamp:if early {ta_stamp} else {fs_stamp}, + value:timeline::stamp((if early {s::STAMP_TA} else {s::STAMP_FRAGMENT})-256,draw-1), + event:u32::from(!early), + }); + self.write_barrier(P::TilerToFragment, sync::Barrier { + stamp: ta_stamp, wait_value: ta_value, event: 0, self_value: fs_value, + uuid: 0x3d0000, dependency: sync::Dependency::TilerToFragment })?; + self.write_barrier(P::TilerDependency, sync::Barrier { + stamp: previous.stamp, wait_value: previous.value, + event: previous.event, self_value: ta_value, + uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass })?; + self.write_barrier(P::FragmentDependency, sync::Barrier { + stamp: fs_stamp, wait_value: timeline::stamp(s::STAMP_FRAGMENT - 256, draw - 1), event: 1, self_value: fs_value, + uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass }) + } + // Prepare each private command after its retired slot is reset. The + // reusable host staging allocation is copied before the next slot is built. + fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,draw:u64)->Result { + use crate::m3_compute_layout::GpuVa; + let pass=&self.passes[self.slot]; + let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); + let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); + let gpu=|field:P| GpuVa::new(pass.get(field).gpu_va()?).map_err(|_|EINVAL); + let seq=pass.get(P::TilerSequence); + let command=pass.get(P::TilerCommand); + let g=Self::geometry(r,self.clusters)?; + if g.tpc_bytes>pass.get(P::Tpc).size() as u64 + || g.tilemap_bytes>pass.get(P::Tilemap).size() as u64 {return Err(E2BIG);} + // The old per-slot register keeps its seed entry, independent of the + // draw-dependent RenderScene reference. Do not change this policy here. + let initial_scene_entry=queue::FirmwareVa::new(self.objects[s::BM_SCENES].va() + +(self.slot as u64+1)*4).map_err(|_|EINVAL)?; + let value=tiler::Command { + context:self._binding.slot(),register_set:self.registers, + notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?, + scene:fw(P::Scene)?,empty:shared(547)?,gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, + sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?, + scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?, + stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], + user_timestamps:[fw(P::TilerUserStart)?,fw(P::TilerUserEnd)?],preemption:[gpu(P::Preemption0)?,gpu(P::Preemption1)?,gpu(P::Preemption2)?], + tilemap:gpu(P::Tilemap)?,tpc:gpu(P::Tpc)?,tpc_bytes:g.tpc_bytes,heap:gpu(P::HeapMetadata)?, + scene_user:gpu(P::SceneUser)?,initial_scene_entry,geometry:g,page_count:PB_PAGES, + vdm:crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, + multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0, + }; + value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?; + self.passes[self.slot].get_mut(P::TilerCommand).write(0,&self.tiler_bytes) + } + fn prepare_sequences(&mut self,draw:u64)->Result { + for fs in [false,true] { + let pass=&self.passes[self.slot]; + let region=|field:P| {let b=pass.get(field);Region::new(b.va(),b.size()).map_err(|_|EINVAL)}; + let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); + let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); + let target=if fs{P::FragmentSequence}else{P::TilerSequence}; + let notifier=&self.objects[NOTIFIER]; + let owners=sequence::Owners { + command:region(if fs{P::FragmentCommand}else{P::TilerCommand})?,sequence:region(target)?, + stats:self.stats,notifier:Region::new(notifier.va(),notifier.size()).map_err(|_|EINVAL)?, + queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?, + manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?, + fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?, + context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw), + }; + let timed=self.stage_timestamps[usize::from(fs)]; + let length=if fs { + let b=pass.get(P::FragmentScratch); + let scratch=GpuRegion::new(b.gpu_va()?,b.size()).map_err(|_|EINVAL)?; + sequence::Fragment{owners,scratch,queue_command_count:timeline::previous(draw)}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)? + }else{sequence::Tiler{owners,scratch:fw(P::TilerScratch)?}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)?}; + self.passes[self.slot].get_mut(target).write(0,&self.sequence_bytes)?; + self.passes[self.slot].get_mut(if fs{P::FragmentCommand}else{P::TilerCommand}).u32(sequence::command_length_offset(fs),length as u32)?; + } + Ok(()) + } + fn view_owner(&mut self,owner:ViewOwner)->&mut Buffer { + match owner {ViewOwner::Shared(index)=>&mut self.objects[index], + ViewOwner::Pass{slot,field}=>self.passes[slot].get_mut(field)} + } + fn patch_context(&mut self)->Result { + let context=self._binding.slot(); + self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT, context)?; + Ok(()) + } + /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster. + fn geometry(r:agx_uapi::UapiRenderCommand,clusters:u32)->Result { + use crate::agx_render::{Geometry,Utile}; + let x=if r.utile_width==16 {Utile::Pixels16} else {Utile::Pixels32}; + let y=if r.utile_height==16 {Utile::Pixels16} else {Utile::Pixels32}; + let mut g=Geometry::new_layered(u32::from(r.width),u32::from(r.height),x,y,clusters,r.layers).map_err(|_|EINVAL)?; + g.set_samples(r.samples).map_err(|_|EINVAL)?; + Ok(g) + } + fn prepare_fragment(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result { + use crate::{m3_compute_layout::GpuVa,agx_render_state::{Program,DepthStencil}}; + let pass=&self.passes[self.slot]; + let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); + let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); + let gpu=|field:P| GpuVa::new(pass.get(field).gpu_va()?).map_err(|_|EINVAL); + let seq=pass.get(P::FragmentSequence); + let command=pass.get(P::FragmentCommand); + let geometry=Self::geometry(r,self.clusters)?; + let mask=if r.flags&16!=0 {u64::MAX} else {u32::MAX as u64}; + let program=|p:agx_uapi::UapiProgram|->Result {Ok(Program { + address:usc.checked_add(u64::from(p.usc&!63)).ok_or(EOVERFLOW)?,resources:p.resource_spec&mask, + })}; + let tilebuffer_control=(u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)).div_ceil(2048) + |(u32::from(r.samples.trailing_zeros() as u8)<<17); + let tile_mode=0x280|u64::from(r.layers>1)|if r.flags&2!=0 {1<<16}else{0}; + let value=fragment::Command { + context:self._binding.slot(),register_set:self.registers, + sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?, + notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:shared(547)?, + gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?, + heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?, + scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?, + fw_stamp:shared(s::FRAGMENT_FW_STAMP)?,stamp_value:timeline::stamp(s::STAMP_FRAGMENT - 256, draw), + timestamps:[fw(P::FragmentStart)?,fw(P::FragmentEnd)?],user_timestamps:[fw(P::FragmentUserStart)?,fw(P::FragmentUserEnd)?], + state:fragment::State { + geometry,multisample:r.multisample_control,merge_upper:[r.merge_upper_x,r.merge_upper_y], + eot:program(r.end_of_tile)?,background:program(r.background)?, + partial_eot:program(r.partial_end_of_tile)?,partial_background:program(r.partial_background)?, + scissor:r.scissor_base,depth_bias:r.depth_bias_base,query:r.occlusion_query_base, + isp_control:0xc000|(r.flags&(1<<18)),tilebuffer_control,mirror_tilebuffer_control:tilebuffer_control, + tile_mode,mirror_tile_mode:tile_mode as u32, + depth:DepthStencil{base:r.depth.base,stride:u64::from(r.depth.stride)}, + stencil:DepthStencil{base:r.stencil.base,stride:u64::from(r.stencil.stride)}, + depth_compression:DepthStencil{base:r.depth.compression_base,stride:u64::from(r.depth.compression_stride)}, + stencil_compression:DepthStencil{base:r.stencil.compression_base,stride:u64::from(r.stencil.compression_stride)}, + zls_control:r.zls_control,depth_dimensions:r.depth_dimensions,depth_clear:r.depth_clear, + stencil_clear:r.stencil_clear,sample_size:u32::from(r.sample_size),process_empty_tiles:r.flags&2!=0, + }, + }; + value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?; + self.passes[self.slot].get_mut(P::FragmentCommand).write(0,&self.fragment_bytes) + } + pub(crate) fn set_user_timestamps(&mut self,addresses:[[u64;2];2])->Result { + if addresses.iter().enumerate().any(|(stage,pair)|*pair!=[0;2] && !self.stage_timestamps[stage]) {return Err(EINVAL);} + for (stage,(command,sequence,offset,start,end)) in [ + (P::TilerCommand,P::TilerSequence,sequence::ta::USER_PAIR,sequence::ta::TIMESTAMP_START,sequence::ta::TIMESTAMP_END), + (P::FragmentCommand,P::FragmentSequence,sequence::fragment::USER_PAIR,sequence::fragment::TIMESTAMP_START,sequence::fragment::TIMESTAMP_END) + ].into_iter().enumerate() { + let pair=addresses[stage]; + if !self.stage_timestamps[stage] {continue;} + self.passes[self.slot].get_mut(command).u64(offset,pair[0])?; + self.passes[self.slot].get_mut(command).u64(offset+8,pair[1])?; + let pointer=if pair==[0,0] {0} else {self.passes[self.slot].get_mut(command).va()+offset as u64}; + self.passes[self.slot].get_mut(sequence).u64(start+sequence::USER_TIMESTAMP_POINTER,pointer)?; + self.passes[self.slot].get_mut(sequence).u64(end+sequence::USER_TIMESTAMP_POINTER,pointer)?; + } + crate::agx_memory::publish();Ok(()) + } + fn at(base:usize,index:usize)->usize {(base+index)%pass_layout::SLOTS} + /// GPU time of the batch of `count` passes starting at pass slot `base`. + pub(crate) fn gpu_ns(&mut self,base:usize,count:usize)->Result { + let start=self.passes[base].get_mut(P::TilerStart).read_u64(0)?; + let end=self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?; + Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + } + /// The batch's first tiler start and last fragment end, in 24 MHz GPU timestamp ticks. + pub(crate) fn gpu_ticks(&mut self,base:usize,count:usize)->Result<(u64,u64)> { + Ok((self.passes[base].get_mut(P::TilerStart).read_u64(0)?, + self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?)) + } + /// Read retained timestamps only after exact batch retirement. The first + /// TA and final fragment delimit the batch in the shared 24 MHz counter. + pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { + if self.batch_count == 0 {return Err(EINVAL);} + Ok([self.passes[self.base].get_mut(P::TilerStart).read_u64(0)?, + self.passes[Self::at(self.base,self.batch_count-1)].get_mut(P::FragmentEnd).read_u64(0)?]) + } + pub(crate) fn stage_ns(&mut self,base:usize,count:usize)->Result<[u64;3]> { + let mut total=[0;3]; + for index in 0..count { + let stages=self.slot_stage_ns(base,count,index)?; + for i in 0..3 {total[i]+=stages[i];} + } + Ok(total) + } + pub(crate) fn slot_stage_ns(&mut self,base:usize,count:usize,index:usize)->Result<[u64;3]> { + if index>=count {return Err(EINVAL);} + let slot=Self::at(base,index); + let duration=|a:u64,b:u64| if a!=0 && b>=a {(b-a)*1000/24} else {0}; + let fs_start=self.passes[slot].get_mut(P::FragmentStart).read_u64(0)?; + let fs_end=self.passes[slot].get_mut(P::FragmentEnd).read_u64(0)?; + let ta_start=self.passes[slot].get_mut(P::TilerStart).read_u64(0)?; + let ta_end=self.passes[slot].get_mut(P::TilerEnd).read_u64(0)?; + Ok([duration(ta_start,ta_end),duration(fs_start,fs_end),duration(ta_end,fs_start)]) + } + pub(crate) fn early_count(&self)->u32 {self.early_count} + pub(crate) fn heads(&self)->[u16;2] {self.heads} + pub(crate) fn ordinal(&self)->u64 {self.draw} + /// The current batch: first pass slot, pass count and last draw number. + pub(crate) fn batch(&self)->(usize,usize,u64) {(self.base,self.batch_count,self.draw)} + /// Whether draws 1..=`draw` have completed both stages (TA and fragment stamps, host and + /// firmware copies, have reached `draw`'s values). Use signed modular ordering + /// across stamp wrap; the runtime bounds outstanding passes to SLOTS. + pub(crate) fn complete_through(&mut self,draw:u64)->Result { + if draw==0 {return Ok(true);} + for (stamp,fw,value) in [(s::TA_STAMP,s::TA_FW_STAMP,timeline::stamp(s::STAMP_TA-256,draw)), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,timeline::stamp(s::STAMP_FRAGMENT-256,draw))] { + if !timeline::reached(self.objects[stamp].read_u32(0)?,value) || !timeline::reached(self.objects[fw].read_u32(0)?,value) {return Ok(false);} + } + Ok(true) + } + /// Whether any draw has been published (the first one initializes the buffer manager). + pub(crate) fn started(&self)->bool {self.draw>0} + pub(crate) fn first(&self)->bool {self.draw==self.batch_count as u64} + /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then + /// the VM must already be bound, no storage may grow and the slots must not be in flight + /// (the caller keeps the in-flight passes plus this batch within SLOTS). + pub(crate) fn begin_batch(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, + commands:&[crate::m3_submit::Command],base:usize,overlap:bool)->Result { + if base>=pass_layout::SLOTS || (!overlap && !self.complete()?) {return Err(EBUSY);} + if overlap { + if !self._binding.matches(vm) {return Err(EBUSY);} + for (index,&command) in commands.iter().enumerate() { + let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let g=Self::geometry(r,self.clusters)?; + let slot=Self::at(base,index); + if g.tpc_bytes>self.passes[slot].get(P::Tpc).size() as u64 + || g.tilemap_bytes>self.passes[slot].get(P::Tilemap).size() as u64 {return Err(EBUSY);} + } + } + if commands.is_empty() || commands.len()>pass_layout::SLOTS {return Err(E2BIG);} + self.draw.checked_add(commands.len() as u64).ok_or(EOVERFLOW)?; + // Validate every pass before changing any queue or completion state. + for &command in commands { + let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + Self::geometry(r,self.clusters)?; + crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?; + } + if !self._binding.matches(vm) { + let next=if let Some(index)=self.cached_views.iter().position(|v|v.binding.matches(vm)) { + self.cached_views.remove(index).map_err(|_|EIO)? + } else { + let binding=uat.bind(vm)?; + let mut views=KVec::new();let mut aliases=KVec::new(); + for i in 0..s::COUNT { + if s::allocation(i).map_err(|_|EINVAL)?.space==Space::ClientGpu { + views.push((ViewOwner::Shared(i),self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; + } + } + for (slot,pass) in self.passes.iter_mut().enumerate() { + for field in pass_layout::FIELDS { + let a=pass_layout::board_allocation(slot,field,self.clusters).map_err(|_|EINVAL)?; + if a.space==Space::ClientGpu {views.push((ViewOwner::Pass{slot,field},pass.get_mut(field).map_gpu_view(vm)?),GFP_KERNEL)?;} + if a.gpu_alias.is_some() {aliases.push(pass.get_mut(field).map_gpu_view(vm)?,GFP_KERNEL)?;} + } + } + CachedViews{binding,aliases,views} + }; + // Finish fallible allocations before replacing the current owner. + let old_views=KVec::with_capacity(next.views.len(),GFP_KERNEL)?; + self.cached_views.reserve(1,GFP_KERNEL)?; + if self.cached_views.len()==3 {drop(self.cached_views.remove(0).map_err(|_|EIO)?);} + let mut old=CachedViews { + binding:core::mem::replace(&mut self._binding,next.binding), + aliases:core::mem::replace(&mut self._aliases,next.aliases), + views:old_views, + }; + for (i,view) in next.views { + old.views.push((i,self.view_owner(i).swap_gpu_view(view)?),GFP_KERNEL)?; + } + self.cached_views.push(old,GFP_KERNEL)?; + } + // Growth is legal only after exact retirement, and before append. + // Cached VM mappings must be dropped before freeing the old backing. + // Partial allocation failure leaves retired queue/stamp state intact; + // already-grown buffers are reusable on the next submission. + for (index,&command) in commands.iter().enumerate() { + let slot=Self::at(base,index); + let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let g=Self::geometry(r,self.clusters)?; + for (field,base,bytes) in [(P::Tpc,s::TPC_LABEL,g.tpc_bytes),(P::Tilemap,s::TILEMAP_LABEL,g.tilemap_bytes)] { + let size=usize::try_from(bytes).map_err(|_|EOVERFLOW)?; + if size>self.passes[slot].get(field).size() { + let buffer=Buffer::tiler(dev,uat.kernel_vm(),vm,size)?; + self.cached_views.clear(); + dev_info!(dev.as_ref(),"M3 render arena growth: slot={} object={} {} -> {} bytes GPU={:#x}\n", + slot,base,self.passes[slot].get(field).size(),size,buffer.gpu_va()?); + *self.passes[slot].get_mut(field)=buffer; + } + } + } + self.checkpoint=(self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base); + self.batch_count=0; + self.base=base; + Ok(()) + } + /// Restore producer state only when no batch is in flight. Running firmware can + /// consume append's WRITE updates without a new doorbell, so overlap cannot abort. + pub(crate) fn abort_batch(&mut self)->Result { + (self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base)=self.checkpoint; + self.objects[s::BM_COUNTER].u32(0,self.draw as u32)?; + self.objects[s::EVENT_COUNT].u32(0,timeline::events(self.draw,2))?; + for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() { + self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; + } + crate::agx_memory::publish(); + Ok(()) + } + /// Append only into caller-reserved free pass slots. With overlap, WRITE updates + /// are immediately visible to running firmware, even before a new doorbell. + /// Each append selects disjoint host-mutated pass storage. A TA dependency + /// retains full fragment-to-next-TA ordering for arbitrary resource hazards. + pub(crate) fn append(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,coalesce_stamps:bool,timestamps:[bool;2])->Result { + self.append_after(r,usc,coalesce_stamps,None,timestamps) + } + pub(crate) fn completion_dependency(&self)->Result { + Ok(sync::CompletionDependency { + stamp:queue::FirmwareVa::new(self.objects[s::FRAGMENT_FW_STAMP].va()).map_err(|_|EINVAL)?, + value:timeline::stamp(s::STAMP_FRAGMENT-256,self.draw),event:1, + }) + } + pub(crate) fn append_after(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64, + coalesce_stamps:bool,predecessor:Option,timestamps:[bool;2])->Result { + if predecessor.is_some_and(|p|p.event!=2) {return Err(EINVAL);} + if self.batch_count>=pass_layout::SLOTS { + pr_err!("M3 render batch limit: batch={} slots={}\n", + self.batch_count,pass_layout::SLOTS); + return Err(E2BIG); + } + self.slot=Self::at(self.base,self.batch_count); + self.stage_timestamps=timestamps; + // Retain the conservative stage-scope permission. Only a pass after the + // first of its batch may overlap its predecessor's stages. Explicit + // start timestamps keep the original full-order execution boundary. + let early=predecessor.is_none() && self.batch_count>0 && *crate::module_parameters::m3_early_tiling.value()!=0 + && r.flags&(1<<5)!=0 && r.vertex_timestamps.start.handle==0 + && r.fragment_timestamps.start.handle==0; + if early {self.early_count+=1;} + // Only reset per-draw objects. Pool allocators, PB lists, queues, + // notification state and previous stamps retain firmware ownership. + self.passes[self.slot].reset(&Self::geometry(r,self.clusters)?)?; + self.init_bm.fill(0)?; // retain the prior shared InitBM reset before encoding + self.draw=self.draw.checked_add(1).ok_or(EOVERFLOW)?; + self.batch_count+=1; + let draw=self.draw; + self.objects[s::BM_COUNTER].u32(0,draw as u32)?; + self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + let entry_offset = (self.draw % u64::from(MANAGER_SCENE_COUNT)) as usize * 4; + let scene_list = &mut self.objects[s::BM_SCENES]; + if entry_offset + 4 > scene_list.size() { return Err(ERANGE); } + let manager_entry = queue::FirmwareVa::new(scene_list.va().checked_add(entry_offset as u64) + .ok_or(EOVERFLOW)?).map_err(|_| EINVAL)?; + let manager_misc = queue::FirmwareVa::new(self.objects[MANAGER_MISC].va()).map_err(|_| EINVAL)?; + let user_buffer = self.passes[self.slot].get_mut(P::SceneUser); + let user_buffer = scene::UserBuffer::new(user_buffer.gpu_va()?, user_buffer.size()).map_err(|_| EINVAL)?; + let mut bytes = [0; scene::SIZE]; + scene::Scene { manager_entry, manager_misc, user_buffer }.encode(&mut bytes).map_err(|_| EINVAL)?; + self.passes[self.slot].get_mut(P::Scene).write(0, &bytes)?; + self.prepare_tiler(r,draw)?; + self.patch_context()?; + self.prepare_fragment(r,usc,draw)?; + self.prepare_sequences(draw)?; + self.prepare_sync(early, draw, predecessor)?; + // Encode before either ring WRITE exposes the command to firmware. + // The caller retains every pass owner until the final flushed stamps, + // queues, events and engine-idle checks retire this entire batch. + if coalesce_stamps { + self.passes[self.slot].get_mut(P::TilerCommand).u32(tiler::FLUSH_STAMPS,0)?; + self.passes[self.slot].get_mut(P::FragmentCommand).u32(fragment::FLUSH_STAMPS,0)?; + } + let dep=P::TilerDependency; + let fs_dep=P::FragmentDependency; + let ta=self.passes[self.slot].get_mut(P::TilerCommand).va(); + let fragment=self.passes[self.slot].get_mut(P::FragmentCommand).va(); + let barrier=self.passes[self.slot].get_mut(P::TilerToFragment).va(); + let dependency=self.passes[self.slot].get(dep).va(); + let init=self.init_bm.va(); + let ta_commands=[init,dependency,ta]; + let fs_commands=[self.passes[self.slot].get(fs_dep).va(),barrier,fragment]; + for (stage,ring,pointers,addresses) in [ + (0,s::TA_RING,s::TA_POINTERS,&ta_commands[usize::from(draw!=1)..]), + (1,s::FRAGMENT_RING,s::FRAGMENT_POINTERS,&fs_commands[usize::from(!early)..])] { + for &address in addresses { + self.objects[ring].u64(usize::from(self.heads[stage])*8,address)?; + self.heads[stage]=(self.heads[stage]+1)%0x500; + } + crate::agx_memory::publish(); + self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; + } + crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();Ok(()) + } + /// Emitted only after both events, stamp/queue checks, clear faults, + /// idle engines and consumed firmware pipes have been checked by Runtime. + /// The existing TA/fragment descriptors request completion stamp flushes. + pub(crate) fn progress(&mut self,dev:&driver::AsahiDevice)->Result { + // Keep summaries sparse at high throughput, but emit at least once a + // second while completed batches advance. Batched draw counts can skip + // multiples of 128; a count-only gate can starve the watchdog observer. + if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + let now = ::ktime_get(); + if self.last_progress_ns != 0 && now - self.last_progress_ns < 1_000_000_000 + && self.draw != 1 && self.draw % 128 != 0 + && !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + self.last_progress_ns = now; + dev_info!(dev.as_ref(),"M3_RETIRED draw={} ta={:x}/{:x} fragment={:x}/{:x} queues={}/{} read={}/{} idle=1 cache_flush=1 events=2 faults=0 ordered_barrier=1 batch={} early={}\n", + self.draw,self.objects[s::TA_STAMP].read_u32(0)?,self.objects[s::TA_FW_STAMP].read_u32(0)?, + self.objects[s::FRAGMENT_STAMP].read_u32(0)?,self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?, + self.objects[s::TA_POINTERS].read_u32(queue::DONE)?,self.objects[s::FRAGMENT_POINTERS].read_u32(queue::DONE)?, + self.objects[s::TA_POINTERS].read_u32(queue::READ)?,self.objects[s::FRAGMENT_POINTERS].read_u32(queue::READ)?,self.batch_count,self.early_count); + Ok(()) + } + pub(crate) fn queues(&self)->[u64;2] {[self.objects[s::TA_QUEUE].va(),self.objects[s::FRAGMENT_QUEUE].va()]} + /// Read retained command bytes only after exact batch retirement. + pub(crate) fn stamp_flush_masks(&mut self,base:usize,count:usize)->Result<[u16;2]> { + if base>=pass_layout::SLOTS || count==0 || count>pass_layout::SLOTS {return Err(EINVAL);} + let mut masks=[0;2]; + for slot in 0..count { + let pass=&mut self.passes[Self::at(base,slot)]; + for (stage,field,offset) in [(0,P::TilerCommand,tiler::FLUSH_STAMPS), + (1,P::FragmentCommand,fragment::FLUSH_STAMPS)] { + if pass.get_mut(field).read_u32(offset)?!=0 {masks[stage]|=1u16<Result<[bool;2]> { + let value=timeline::stamp(s::STAMP_FRAGMENT - 256,self.draw); + Ok([self.objects[s::FRAGMENT_STAMP].read_u32(0)?==value, + self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?==value]) + } + /// Observe each completion condition across both render queues independently. + pub(crate) fn completion_parts(&mut self)->Result<[bool;4]> { + let mut done=[true;4]; + for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] { + done[0]&=self.objects[stamp].read_u32(0)?==value; + done[1]&=self.objects[fw].read_u32(0)?==value; + done[2]&=self.objects[pointers].read_u32(queue::DONE)?==u32::from(head); + done[3]&=self.objects[pointers].read_u32(queue::READ)?==u32::from(head); + } + Ok(done) + } + pub(crate) fn complete(&mut self)->Result { + for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] { + if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value + || self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head) + || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} + } Ok(true) + } + pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + // Retain allocator state as well as payloads: an exhausted parameter + // pool can stall TA without producing an MMU fault. + for (index,bytes) in [(BLOCK_CONTROL,64),(s::BM_COUNTER,64),(MANAGER_MISC,64),(BUFFER_MANAGER,172)] { + for offset in (0..bytes).step_by(4) { + let value=self.objects[index].read_u32(offset)?; + dev_info!(dev.as_ref(),"M3_PB object={} offset={:#x} value={:#x}\n",index,offset,value); + } + } + dev_info!(dev.as_ref(),"M3 fault render VM={} slot={} root={:#x} draw={} batch={}\n",self._binding.vm_id(),self._binding.slot(),self._binding.root(),self.draw,self.batch_count); + for index in 0..self.batch_count { + let slot=Self::at(self.base,index); + let command=self.passes[slot].get_mut(P::TilerCommand); + for index in 0..tiler::REGISTER_COUNT { + let offset=tiler::REGISTERS+index*tiler::REGISTER_STRIDE; + if command.read_u32(offset)?==0x1c880 { + dev_info!(dev.as_ref(),"M3 fault pass slot={} compact_vdm={:#x}\n",slot,command.read_u64(offset+4)?); + } + } + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + // Preserve historic diagnostic IDs while selecting named owners. + for (index,bytes) in s::fault_buffers() { + let b=match index {s::TILEMAP_LABEL=>self.passes[0].get_mut(P::Tilemap), + s::HEAP_METADATA_LABEL=>self.passes[0].get_mut(P::HeapMetadata),_=>&mut self.objects[index]}; + for offset in (0..bytes).step_by(16) { + let lo=b.read_u64(offset)?; + let hi=b.read_u64(offset+8)?; + if lo!=0 || hi!=0 {dev_info!(dev.as_ref(),"M3_BUFFER object={} offset={:#x} {:016x} {:016x}\n",index,offset,lo,hi);} + } + } + } + for (name,stamp,fw,pointers) in [("TA",s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS), + ("fragment",s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS)] { + dev_info!(dev.as_ref(),"M3 {} stamps={:#x}/{:#x} done={} read={}\n",name, + self.objects[stamp].read_u32(0)?,self.objects[fw].read_u32(0)?, + self.objects[pointers].read_u32(queue::DONE)?,self.objects[pointers].read_u32(queue::READ)?); + } Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_resources.rs b/drivers/gpu/drm/asahi/m3/m3_resources.rs new file mode 100644 index 0000000000000..bfc94fdfa0ca8 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_resources.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M3 GPU resources admitted against the measured boot contract (J514S on T6030). +pub(crate) use crate::agx_resources::{Region, Resources}; + +use kernel::{bindings, c_str, io::resource::Resource, of, prelude::*}; + +/// Live overlays are not inserted in OF's boot-time reserved_mem registry. +/// Keep the ordinary lookup for every static handoff. A no-map node is +/// accepted only when its range is outside all Linux System RAM. +pub(crate) fn reserved_resource(node: &of::Node, name: &CStr) -> Result { + if let Ok(r) = node.reserved_mem_region_to_resource_byname(name) { + return Ok(r); + } + if let Some(found) = crate::m3_board::static_region(node, name) { + return found.map(|(r, _)| r); + } + // Only for a GPU node whose compatible is exactly that of an M3 SoC table. + let compatible: KVec = node.get_property(c_str!("compatible"))?; + let compatible = compatible.strip_suffix(b"\0").ok_or(EINVAL)?; + if !crate::m3_soc::SOCS.iter().any(|soc| compatible == soc.gpu.as_bytes()) { + return Err(EINVAL); + } + let names: KVec = node.get_property(c_str!("memory-region-names"))?; + let index = names.split(|b| *b == 0).position(|n| n == name.to_bytes()).ok_or(EINVAL)?; + let region = node.parse_phandle(c_str!("memory-region"), index).ok_or(EINVAL)?; + let nomap: KVec = region.get_property(c_str!("no-map"))?; + let reg: KVec = region.get_property(c_str!("reg"))?; + if !nomap.is_empty() || reg.len()!=2 { return Err(EINVAL); } + let (base,size)=(reg[0],reg[1]); + if base < 1<<40 || size == 0 || (base|size)&0x3fff != 0 + || base.checked_add(size).ok_or(EOVERFLOW)? > 1<<42 { return Err(EINVAL); } + // SAFETY: read-only resource-tree query with checked numeric bounds. + if unsafe { bindings::region_intersects(base, size.try_into()?, bindings::IORESOURCE_SYSTEM_RAM as _, 0) } != bindings::REGION_DISJOINT as i32 { + return Err(EBUSY); + } + let raw = bindings::resource { + start: base, end: base+size-1, flags: bindings::IORESOURCE_MEM as _, + ..Default::default() + }; + // SAFETY: Resource is repr(transparent) over Opaque, which + // preserves layout. The initialized descriptor is owned, with no pointers + // into temporary data; its range is checked above. + Ok(unsafe { core::mem::transmute::(raw) }) +} + +pub(crate) fn from_device( + pdev: &kernel::platform::Device, + soc: &crate::m3_soc::Soc, +) -> kernel::error::Result { + crate::m3_board::admit(pdev, soc) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs new file mode 100644 index 0000000000000..3e8cbbfa93781 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! RTKit callbacks for the identified J514S firmware. The firmware advertises +//! its crash storage by physical address inside the reserved data segment. + +use crate::agx_resources::Region; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use kernel::{ + c_str, impl_has_hr_timer, + device::Core, + sync::Completion, + io::mem::{Mem, MemFlag}, + iosys_map::IoSysMapRef, + platform, + prelude::*, + soc::apple::rtkit, + sync::{aref::ARef, Arc, ArcBorrow}, + time::{Delta, Monotonic, hrtimer::{HrTimer, HrTimerCallback, + HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode}}, +}; + +/// IRQ-safe notification tokens for the single GPU completion worker. +#[pin_data] +pub(crate) struct EventWait { + count: AtomicU64, + wait_us: AtomicU64, + overlap_cpu: AtomicBool, + cpu_preparations: AtomicU64, + sleeps: AtomicU64, + timer_wakes: AtomicU64, + #[pin] + arrived: Completion, + #[pin] + timer: HrTimer, +} + +impl EventWait { + fn new() -> Result> { + Arc::pin_init( + pin_init!(EventWait { + count: AtomicU64::new(0), + wait_us: AtomicU64::new(100), + overlap_cpu: AtomicBool::new(crate::m3_params::unlocked_wait()), + cpu_preparations: AtomicU64::new(0), + sleeps: AtomicU64::new(0), + timer_wakes: AtomicU64::new(0), + arrived <- Completion::new(), + timer <- HrTimer::new(), + }), + GFP_KERNEL, + ) + } + + fn record(&self) { + self.count.fetch_add(1, Ordering::Release); + self.arrived.complete(); + } + + /// Diagnostic tuning changes only the bounded resnapshot cadence, never + /// stamps, pipe retirement, event counts, fences or the two-second timeout. + /// Atomic publication permits root to compare policies in the same boot. + pub(crate) fn set_wait_us(&self, us: u64) -> Result { + if !(25..=1000).contains(&us) { return Err(EINVAL); } + self.wait_us.store(us, Ordering::Relaxed); + Ok(()) + } + + pub(crate) fn cpu_overlap(&self) -> bool { + self.overlap_cpu.load(Ordering::Relaxed) + } + pub(crate) fn set_cpu_overlap(&self, enabled: bool) { + self.overlap_cpu.store(enabled, Ordering::Relaxed); + } + + pub(crate) fn record_cpu_preparation(&self) { + self.cpu_preparations.fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn snapshot(&self) -> (u64, u64, u64, u64, u64) { + (self.wait_us.load(Ordering::Relaxed), self.count.load(Ordering::Acquire), + self.sleeps.load(Ordering::Relaxed), self.timer_wakes.load(Ordering::Relaxed), + self.cpu_preparations.load(Ordering::Relaxed)) + } + + /// Sleep until a notification later than `seen` has been recorded or + /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250, + /// even when the GPU stamp arrives just after its notification. The timer + /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting. + /// Only the single completion worker may call this method. + pub(crate) fn wait_past(self: &Arc, seen: u64) -> bool { + // Events can arrive while the worker is polling. Discard stale wakeup + // tokens before checking the sequence. A concurrent event either moves + // the sequence or leaves a token, including between the check and wait. + // Bound draining so a busy interrupt source cannot hold this worker. + for _ in 0..64 { + if !self.arrived.try_wait_for_completion() { + if self.count.load(Ordering::Acquire) != seen { return true; } + // The handle keeps EventWait alive and cancels synchronously + // on every return, before a subsequent wait can arm the timer. + self.sleeps.fetch_add(1, Ordering::Relaxed); + let timer = self.clone().start(Delta::from_micros( + self.wait_us.load(Ordering::Relaxed) as i64)); + // Retain one tick as a backup bound; normally either the real + // interrupt or our high-resolution timer supplies the token. + self.arrived.wait_for_completion_timeout(1); + drop(timer); + return self.count.load(Ordering::Acquire) != seen; + } + } + // Resnapshot GPU progress before draining another batch of tokens. + true + } +} + +impl HrTimerCallback for EventWait { + type Pointer<'a> = Arc; + + fn run(this: ArcBorrow<'_, Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { + // Wake only: a timer expiry is not a firmware completion notification. + this.timer_wakes.fetch_add(1, Ordering::Relaxed); + this.arrived.complete(); + HrTimerRestart::NoRestart + } +} + +impl_has_hr_timer! { + impl HasHrTimer for EventWait { + mode: RelativeMode, field: self.timer + } +} + +pub(crate) struct State { + dev: ARef, + drm: crate::driver::AsahiDevRef, + data: Region, + pub(crate) health: Arc, + claimed: AtomicBool, + /// Firmware event notifications received on the application endpoint. + /// The firmware sends one after a command's completion processing. + pub(crate) event_messages: AtomicU64, + pub(crate) events: Arc, +} + +/// Shared status contains no device references, so DRM files may retain it +/// after runtime teardown without retaining MMIO or creating a DRM cycle. +pub(crate) struct Health { + crashed: AtomicBool, + mapped: AtomicBool, + failed: AtomicBool, + progress: crate::agx_host_progress::Progress, + pub(crate) timing: crate::agx_timing_stats::Stats, +} + +impl Health { + pub(crate) fn healthy(&self) -> bool { + self.mapped.load(Ordering::Acquire) + && !self.crashed.load(Ordering::Acquire) + && !self.failed() + } + + pub(crate) fn set_gpu_pending(&self, pending: bool) { self.progress.set_pending(pending); } + + pub(crate) fn activity_snapshot(&self) -> (u64, u64, bool) { + let (generation, epoch) = self.progress.activity_snapshot(); + (generation, epoch, self.healthy()) + } + + pub(crate) fn record_completion(&self) { self.progress.record_completion(); } + + pub(crate) fn progress_snapshot(&self) -> (u64, u64, u64, bool) { + let (generation, count, timestamp) = self.progress.snapshot(); + (generation, count, timestamp, self.healthy()) + } + + pub(crate) fn failed(&self) -> bool { self.failed.load(Ordering::Acquire) } + + /// Whether RTKit reported a firmware crash. + pub(crate) fn crashed(&self) -> bool { self.crashed.load(Ordering::Acquire) } + + pub(crate) fn mark_failed(&self) { self.failed.store(true, Ordering::Release); } +} + +impl State { + pub(crate) fn new(dev: &platform::Device, drm: crate::driver::AsahiDevRef, data: Region) -> Result> { + Ok(Arc::new( + Self { + dev: dev.into(), + drm, + data, + health: Arc::new(Health { + crashed: AtomicBool::new(false), + mapped: AtomicBool::new(false), + failed: AtomicBool::new(false), + progress: crate::agx_host_progress::Progress::new(), + timing: crate::agx_timing_stats::Stats::new(), + }, GFP_KERNEL)?, + claimed: AtomicBool::new(false), + event_messages: AtomicU64::new(0), + events: EventWait::new()?, + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn healthy(&self) -> bool { + self.health.healthy() + } +} + +pub(crate) struct FirmwareBuffer { + state: Arc, + mapping: Mem, + physical: usize, + offset: usize, + size: usize, +} + +impl Drop for FirmwareBuffer { + fn drop(&mut self) { + // Includes allocation failure in RTKit's wrapper after shmem_map + // returned: readiness must not survive loss of the actual mapping. + self.state.health.mapped.store(false, Ordering::Release); + } +} + +impl rtkit::Buffer for FirmwareBuffer { + fn iova(&self) -> Result { + Ok(self.physical) + } + fn buf(&mut self) -> Result> { + self.mapping.iosys_map(self.offset, self.size) + } +} + +pub(crate) struct Operations; + +#[kernel::macros::vtable] +impl rtkit::Operations for Operations { + type Data = Arc; + type Buffer = FirmwareBuffer; + + fn crashed(state: ArcBorrow<'_, State>, crashlog: Option<&[u8]>) { + state.health.crashed.store(true, Ordering::Release); + // Wake the serialized worker even if the firmware can no longer send + // its usual completion notification. Never take the runtime lock here. + state.events.record(); + crate::m3_completion::queue(&state.drm); + dev_err!( + state.dev.as_ref(), + "M3 G15S: firmware crashed, retained crashlog bytes={}\n", + crashlog.map_or(0, |b| b.len()) + ); + if let Some(bytes) = crashlog { + crate::t6031_start::capture_crashlog(state.dev.as_ref(), bytes); + } + } + + fn shmem_map( + state: ArcBorrow<'_, State>, + physical: usize, + size: usize, + ) -> Result { + // The only system buffer advertised by this admitted firmware is its + // preallocated crash log. Never reinterpret an arbitrary DVA as a PA. + let offset = physical + .checked_sub(state.data.base as usize) + .ok_or(EINVAL)?; + if size == 0 + || (physical | size) & 0xfff != 0 + || offset.checked_add(size).ok_or(EOVERFLOW)? > state.data.size as usize + { + return Err(EINVAL); + } + if state.claimed.swap(true, Ordering::AcqRel) { + return Err(EBUSY); + } + let node = state.dev.as_ref().of_node().ok_or(ENODEV)?; + let resource = crate::m3_resources::reserved_resource(&node,c_str!("fw-data"))?; + if resource.start() != state.data.base || resource.size() != state.data.size { + return Err(EINVAL); + } + // SAFETY: The validated no-map data reservation belongs to this + // firmware session. This CPU mapping is retained by RTKit until ASC + // is stopped and callbacks drained; physical pages are never freed. + // An uncached CPU alias lets RTKit snapshot newly written crash data. + let mapping = unsafe { Mem::try_new(resource, MemFlag::WC.into()) }?; + dev_info!( + state.dev.as_ref(), + "M3 G15S: mapped firmware crash storage PA={:#x} size={:#x}\n", + physical, + size + ); + state.health.mapped.store(true, Ordering::Release); + Ok(FirmwareBuffer { + state: state.into(), + mapping, + physical, + offset, + size, + }) + } + + fn recv_message(state: ArcBorrow<'_, State>, endpoint: u8, message: u64) { + // Firmware retries its event notification until the shared rings are + // drained, including after the final userspace job. Never take the + // runtime mutex in the RTKit callback: enqueue the serialized worker. + if endpoint == 0x20 && message == 0x0042_0000_0000_0000 { + // Record the IRQ-safe token before publishing the external count. + // A concurrent waiter sees either the moved sequence or a token. + state.events.record(); + state.event_messages.fetch_add(1, Ordering::Release); + if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + dev_info!(state.dev.as_ref(), "M3 G15S_TIMING event monotonic_ns={} messages={}\n", + ::ktime_get(), + state.event_messages.load(Ordering::Acquire)); + } + crate::m3_completion::queue(&state.drm); + return; + } + dev_warn!( + state.dev.as_ref(), + "M3 G15S: RTKit application ep={:#x} message={:#018x}\n", + endpoint, + message + ); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs new file mode 100644 index 0000000000000..08885f591c140 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -0,0 +1,1246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +use core::{mem::ManuallyDrop, sync::atomic::Ordering}; +use kernel::{device::Core, platform, prelude::*, soc::apple::rtkit, sync::Arc, + time::{Instant, Monotonic, Delta, delay::fsleep}}; +use crate::{driver, mmu, m3_device::Device, m3_config::Config, m3_rtkit}; + +#[path = "m3_batch_policy.rs"] +pub(crate) mod policy; +const _: () = assert!(policy::SLOTS == crate::m3_pass_layout::SLOTS); + +enum NativeJob {Compute(crate::m3_compute::Compute),Render(crate::m3_render::Render)} +/// Completion-worker outcome. +pub(crate) enum Service {Idle,Progress,Waiting(u64)} +/// The published batch: its commands in order, as (packet, command index). +struct Batch { + entries: KVec<(Arc,usize)>, + index: usize, kind: usize, first: crate::m3_submit::Command, + preparation_ns: i64, previous_events: u64, expected_events: u64, + start: Instant, measure: bool, doorbell_counter_ns: i64, + /// Render: first pass slot and the batch's last draw number. + base: usize, last_draw: u64, publication_tick:u64, + polling_ns: i64, polls: u64, stamp_ns: i64, first_event_ns: i64, events_ns: i64, + completion_ns: [i64;4], + /// First successful stamp observations and the preceding incomplete reads, in 24 MHz ticks. + fragment_host_tick:u64, fragment_fw_tick:u64, + fragment_host_pending_tick:u64, fragment_fw_pending_tick:u64, + pipes_idle_tick:u64, + coalesced_stamps: u64, + mixed: Option, +} +struct MixedBatch {compute_index:usize, renders:usize, computes:usize, dependencies:usize} +#[derive(Clone, Copy, Default)] +struct GeometryTiming { + width: u16, height: u16, samples: u8, count: u64, + prepare_ns: i64, active_ns: i64, gpu_ns: u64, + tiling_ns: u64, fragment_ns: u64, max_fragment_ns: u64, + polling_ns: i64, retirement_ns: i64, sleeping_ns: i64, polls: u64, + stamp_ns:i64, first_event_ns:i64, events_ns:i64, pickup_ns:i64, tail_ns:i64, + completion_ns: [i64;4], +} +impl NativeJob { + fn complete(&mut self)->Result {match self {Self::Compute(j)=>j.complete(),Self::Render(j)=>j.complete()}} + fn log(&mut self,dev:&driver::AsahiDevice)->Result {match self {Self::Compute(j)=>j.log(dev),Self::Render(j)=>j.log(dev)}} + fn batch_gpu_span(&mut self)->Result<[u64;2]> {match self {Self::Compute(j)=>j.batch_gpu_span(),Self::Render(j)=>j.batch_gpu_span()}} +} +struct Inner { + transport: rtkit::RtKit, + state: Arc, + jobs: KVec, + packets: KVec>, + /// Packets not yet (fully) published: (packet, next command index), most urgent first. + pending: KVec<(Arc,usize)>, + /// Published batches, oldest first. More than one only with asahi.m3_pipeline_depth > 1. + active: KVec, + gpu_pending: bool, + active_power_last: Option>, + active_power_samples: u32, + fault_captured: bool, + timing: [[i64;7];2], + render_batches: [u64;crate::m3_pass_layout::SLOTS], + geometry: [GeometryTiming; 32], + geometry_overflow: u64, + /// Whether a T8122 job verdict line was already emitted for the current job, so the + /// execute-path fallback does not add a second one. T8122 only. + t8122_verdict: bool, + /// The step the firmware boot sequence has reached, for the T8122 boot verdict. T8122 only. + boot_step: crate::t8122_start::BootStep, + /// Packets that joined a batch started by an earlier packet (debug summary). + coalesced_packets: u64, + coalesced_render_stamps: u64, + mixed_batches:u64, + /// Recent measured render GPU time per pass by VM. Smoothed over batches; + /// replacing the least recently measured VM does not retain its mappings. + pass_cost: policy::PassCosts, + /// Latched only after the isolated InitBM draw fully retired. + render_initialized: bool, + config: Config, + uat: mmu::Uat, + drm: driver::AsahiDevRef, + device: Device, +} +impl Inner { + /// Latch before allocating diagnostics, once per runtime. Called under + /// its mutex even for errors outside the active-job polling loop. + fn capture_fault(&mut self, primary: Error) { + self.state.health.mark_failed(); + if self.fault_captured { return; } + self.fault_captured = true; + #[cfg(CONFIG_DEV_COREDUMP)] + if let Err(error) = self.config.fault_snapshot(&self.drm, primary, self.gpu_pending) { + dev_err!(self.drm.as_ref(), "M3 fault snapshot failed: {:?}\n", error); + } + } + + /// Record a batch that failed or never retired. Mark the GPU failed first, + /// so that a failing diagnostic read cannot leave a job that never retired + /// behind a device that still reports itself healthy. + fn fail(&mut self, index: usize, vm: &mmu::Vm, primary: Error) { + self.capture_fault(primary); + if crate::t8122_start::is_t8122(self.device.soc()) { + let pstate=self.device.pstate_register().ok(); + // The job's first GPU start timestamp: nonzero means the firmware dispatched it, + // whatever the power state reads at the timeout. + let start_stamp=self.jobs[index].batch_gpu_span().ok().map(|span| span[0]); + crate::t8122_start::job_failed_verdict(self.drm.as_ref(),primary,start_stamp,pstate,self.state.health.crashed()); + self.t8122_verdict=true; + } else if crate::t6031_start::is_t6031(self.device.soc()) { + crate::t6031_start::job_failed_verdict(self.drm.as_ref(), primary, self.state.health.crashed()); + self.t8122_verdict=true; + } + let events=self.state.event_messages.load(Ordering::Acquire); + if let Err(e) = self.jobs[index].log(&self.drm) { + dev_err!(self.drm.as_ref(), "M3 job diagnostics failed: {:?}\n", e); + } + if let Err(e) = self.device.log_engine_state(vm) { + dev_err!(self.drm.as_ref(), "M3 engine diagnostics failed: {:?}\n", e); + } + let _=self.config.log_recovery_state(&self.drm,events); + if crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault) { + if let Err(e)=self.resume_experiment(index, vm) { + dev_err!(self.drm.as_ref(),"M3: resume-after-fault: stopped by {:?}\n",e); + } + } + } + /// asahi.g15_debug bit 56. Wait up to 100 ms for the firmware to report + /// that it halted, resume it, and log every change of the halt words, + /// the engine state, the failed job's retirement, whether every queue + /// message was consumed and the event count for about half a second, + /// then the event ring entries the host has not consumed. The GPU stays + /// marked failed; the event ring's read index is not advanced. + fn resume_experiment(&mut self, index: usize, vm: &mmu::Vm) -> Result { + let start=Instant::::now(); + let mut halt=self.config.halt_state()?; + while halt.1==0 && start.elapsed()::now(); + let mut last=None; + for _ in 0..500 { + let halt=self.config.halt_state()?; + // 0: idle and no fault bits, 1: an engine is busy, 2: fault bits set. + let engines=match self.device.check_idle() {Ok(())=>0u8,Err(e) if e==EBUSY=>1,Err(_)=>2}; + let complete=self.jobs[index].complete().unwrap_or(false); + // 0: queues not drained, 1: every submitted queue message consumed, 2: read failed. + let pipes=match self.config.pipes_idle() {Ok(false)=>0u8,Ok(true)=>1,Err(_)=>2}; + let events=self.state.event_messages.load(Ordering::Acquire); + let crashed=self.state.health.crashed(); + let now=(halt,engines,complete,pipes,events,crashed); + if last!=Some(now) { + dev_info!(self.drm.as_ref(),"M3: resume-after-fault: +{}us: halt_count={} halted={} resume={} engines={} job_complete={} pipes_idle={} event_messages={} crashed={}\n", + resumed.elapsed().as_nanos()/1000,halt.0,halt.1,halt.2,engines,complete,pipes,events,crashed); + last=Some(now); + } + if crashed {break;} + fsleep(Delta::from_millis(1)); + } + self.jobs[index].log(&self.drm)?; + self.device.log_engine_state(vm)?; + self.config.log_event_ring(&self.drm)?; + let events=self.state.event_messages.load(Ordering::Acquire); + self.config.log_recovery_state(&self.drm,events) + } +} +impl Inner { + /// Fail every packet that was never published. Their fences signal with the error. + fn fail_pending(&mut self, error: Error) { + for (packet,_) in self.pending.drain_all() {packet.finish(Err(error));} + } + /// A published batch failed or never retired: latch the fault, log it, and fail its + /// packets. The packets stay retained (with their VM job guards) until teardown, because + /// the firmware has not provably stopped using their memory. + fn fail_active(&mut self, error: Error) { + let _=self.config.performance_request(self.device.soc().chip_id,false); + if let Some((index,vm))=self.active.first().map(|b|(b.index,b.entries[0].0.vm.clone())) { + self.log_owners(error); + self.fail(index,&vm,error); + } else {self.capture_fault(error);} + for batch in self.active.drain_all() {Self::finish_all(&batch.entries,Err(error));} + self.fail_pending(error); + } + /// Name the processes whose published commands were in flight when the GPU failed. Batches + /// in flight share one VM, so this is normally one process; a VM shared between processes + /// names each submitter once (the first four). + fn log_owners(&self, error: Error) { + let commands:usize=self.active.iter().map(|b|b.entries.len()).sum(); + let mut logged=[0i32;4]; + let mut count=0; + for (packet,_) in self.active.iter().flat_map(|b|b.entries.iter()) { + let owner=&packet.owner; + if logged[..count].contains(&owner.pid) {continue;} + if count==logged.len() {break;} + dev_err!(self.drm.as_ref(),"M3 failed work owner: VM={} process {} pid {} thread {} (error {:?}, {} commands in flight)\n", + packet.vm.id(),owner.process(),owner.pid,owner.thread(),error,commands); + logged[count]=owner.pid;count+=1; + } + } + /// Hardware completion is separate from fence signaling: cancellation can win the + /// signal race while the DMA guard must survive until its final published batch retires. + fn packet_active(&self, packet:&Arc)->bool { + self.active.iter().any(|b|b.entries.iter().any(|(p,_)|Arc::ptr_eq(p,packet))) + } + fn retire_packets(&mut self,entries:&[(Arc,usize)]) { + for (packet,_) in entries { + let pending=self.pending.iter().any(|(p,_)|Arc::ptr_eq(p,packet)); + if policy::packet_retired(pending,self.packet_active(packet)) {packet.retired();} + } + // Keep exactly active DMA owners. Successful retirement may precede a VM switch. + self.packets.retain(|p| self.active.iter().any(|b|b.entries.iter().any(|(q,_)|Arc::ptr_eq(p,q)))); + } + fn fail_front(&mut self,error:Error) { + if let Ok((packet,_))=self.pending.remove(0) { + if !self.packet_active(&packet) && !self.fault_captured {packet.release_guard();} + packet.finish(Err(error)); + } + } + /// Signal every packet of `entries` (packets that continue past the batch included). + fn finish_all(entries:&[(Arc,usize)],result:Result) { + for (packet,_) in entries {packet.finish(result);} + } + /// Choose the next batch from the front of `pending`: the front packet's next commands of + /// one engine, and for render also the leading render commands of following packets of the + /// same VM, up to the ordered-batch limit. + fn next_batch(&self,max:usize)->Result,usize)>> { + let mut entries=KVec::new(); + let Some((packet,next))=self.pending.first() else {return Ok(entries);}; + let mixed=policy::mixed_prefix(crate::m3_params::mixed_batch(), + self.device.soc().chip_id,self.render_initialized,self.active.is_empty(), + packet.commands[*next..].iter().map(|c|matches!(c,crate::m3_submit::Command::Render{..}))); + if mixed>0 { + for index in *next..*next+mixed {entries.push((packet.clone(),index),GFP_KERNEL)?;} + return Ok(entries); + } + let render=matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}); + let limit=if render { + // The first render draw initializes the shared buffer manager (InitBM) and opens + // both queues. Publish it alone: appending later draws first advances the shared + // manager and event counters it starts from, and on J613 the second pass's + // fragment then fails with firmware error event 4 (draw=2 batch=2). + let started=self.jobs.iter().any(|job| matches!(job,NativeJob::Render(j) if j.started())); + if started {crate::m3_params::render_batch_size().min(max).max(1)} else {1} + } else { + crate::m3_params::compute_batch_size() + .clamp(1,crate::m3_compute_storage::SLOTS) + }; + let same_engine=|c:&crate::m3_submit::Command| render==matches!(c,crate::m3_submit::Command::Render{..}); + let per_pass=self.pass_cost.get(packet.vm.id()); + let budget=crate::m3_params::render_batch_budget_ns(self.device.soc()); + let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); + for (position,(candidate,start)) in self.pending.iter().enumerate() { + // Later packets join only render batches, only from their first command, and only + // for the VM the batch is bound to, while the batch's expected GPU time is within + // the budget. + if position>0 && !policy::may_join(render,*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency) {break;} + let mut index=*start; + while index=limit || indexResult> { + let Some(last)=self.active.last() else {return Ok(Some((0,crate::m3_pass_layout::SLOTS)));}; + if !self.render_initialized {return Ok(None);} + let Some((packet,next))=self.pending.first() else {return Ok(None);}; + if !matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}) + || self.active.iter().any(|b| b.kind!=0) || !packet.vm.same(&last.entries[0].0.vm) {return Ok(None);} + let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); + let Some(room)=policy::pipeline_room(crate::m3_params::pipeline_depth(self.device.soc()), + self.active.len(),in_flight,last.base,last.entries.len()) else {return Ok(None);}; + // Yield at the next retirement when more urgent work is already waiting. + if policy::urgent_waiting(self.pending.iter().map(|(p,_)|p.urgency), + last.entries[0].0.urgency) {return Ok(None);} + if !policy::within_budget(self.pass_cost.get(packet.vm.id()), + crate::m3_params::render_batch_budget_ns(self.device.soc()),in_flight) {return Ok(None);} + if !policy::overlap_ready(self.render_initialized,true,true,true, + self.config.pipe_free(0)? && self.config.pipe_free(1)?) {return Ok(None);} + Ok(Some(room)) + } + /// Publish pending batches while allowed: one when none is active, more behind it with + /// asahi.m3_pipeline_depth > 1. Returns whether anything was published or failed. + fn publish_next(&mut self)->bool { + let mut progress=false; + while !self.pending.is_empty() && self.state.healthy() { + let overlap=!self.active.is_empty(); + let (base,max)=match self.pipeline_slot() { + Ok(Some(slot))=>slot, + Ok(None)=>break, + Err(error)=>{self.fail_active(error);return true;} + }; + let entries=match self.next_batch(max) { + Ok(entries) if !entries.is_empty()=>entries, + _=>{self.fail_front(ENOMEM);progress=true;continue;}, + }; + match self.publish(&entries,base,overlap) { + Ok(batch)=>{ + // Consume the published commands from `pending`. + let Some((last,last_index))=entries.last().map(|(p,i)|(p.clone(),*i)) else {break;}; + while let Some((packet,_))=self.pending.first() { + if Arc::ptr_eq(packet,&last) { + if last_index+1==last.commands.len() {let _=self.pending.remove(0);} + else {self.pending[0].1=last_index+1;} + break; + } + let _=self.pending.remove(0); + } + if self.active.push(batch,GFP_KERNEL).is_err() { + // Published but untracked: it can never be retired. + self.capture_fault(ENOMEM);Self::finish_all(&entries,Err(ENOMEM)); + self.fail_active(ENOMEM);return true; + } + progress=true; + } + Err((error,published))=>{ + if published { + if crate::t8122_start::is_t8122(self.device.soc()) && !self.t8122_verdict { + crate::t8122_start::job_setup_failed_verdict(self.drm.as_ref(),error); + self.t8122_verdict=true; + } else if crate::t6031_start::is_t6031(self.device.soc()) && !self.t8122_verdict { + crate::t6031_start::job_setup_failed_verdict(self.drm.as_ref(), error); + self.t8122_verdict=true; + } + // A live ring WRITE or doorbell may have been published: keep packets + // retained (self.packets) and fail them and everything in flight. + self.capture_fault(error); + Self::finish_all(&entries,Err(error)); + self.fail_active(error); + self.fail_pending(error); + return true; + } else if overlap && error==EBUSY { + // Not publishable behind the running batch (new VM binding or storage + // growth): wait for it to retire. + break; + } else { + // Nothing reached the firmware: fail only the front packet, as a failed + // execute() did before publication. + self.fail_front(error); + progress=true; + } + } + } + } + progress + } + /// Prepare and publish one batch. On error, reports whether firmware may see its writes. + fn publish(&mut self,entries:&[(Arc,usize)],base:usize,overlap:bool)->core::result::Result { + let render=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Render{..})); + let compute=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Compute(_))); + if render && compute {return self.publish_mixed(entries,base,overlap);} + let early=|e:Error|(e,false); + let (packet,first_index)=(&entries[0].0,entries[0].1); + let control=packet.commands[first_index]; + let batch_count=entries.len(); + let prepare=Instant::::now(); + // All host bookkeeping is reserved before mutating shared producer state. + // No allocation may fail after publication and lose a DMA owner. + self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + self.packets.reserve(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut owned=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + for (p,i) in entries {owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} + + // Retain the batch's packets (and their VM job guards) until it retires; with batches + // in flight, also those of the earlier ones. + if !overlap {self.packets.clear();} + for (p,_) in entries { + if !self.packets.iter().any(|q| Arc::ptr_eq(q,p)) {self.packets.push(p.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} + } + let found=self.jobs.iter().position(|job| matches!((job,control), + (NativeJob::Compute(_),crate::m3_submit::Command::Compute(_)) | + (NativeJob::Render(_),crate::m3_submit::Command::Render{..}))); + let index=found.unwrap_or(self.jobs.len()); + if found.is_none() { + let job=match control { + crate::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), + crate::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), + }; + self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + } else { + match (&mut self.jobs[index],control) { + (NativeJob::Render(_),crate::m3_submit::Command::Render{..})=>{}, + (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>j.replay(&self.uat,&packet.vm,c).map_err(early)?, + _=>return Err(early(ENOTSUPP)), + } + } + if let NativeJob::Compute(j)=&mut self.jobs[index] { + let prepared=(||->Result { + for (slot,(p,next)) in entries.iter().enumerate() { + if slot!=0 { + let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + j.append(c)?; + } + let timestamps=p.timestamp_addresses(*next); + let coalesce=crate::m3_compute_storage::coalesce_stamp_flush( + p.wide_visibility[*next],timestamps,slot+1==batch_count); + j.prepare_completion(coalesce)?; + j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.attachments[*next])?; + } + Ok(()) + })(); + if let Err(error)=prepared { + j.abort_batch().map_err(|e|(e,true))?; + if found.is_none() {drop(self.jobs.remove(index).map_err(|_|early(EIO))?);} + return Err(early(error)); + } + } + let mut coalesced_stamps=0; + let coalesce_render=crate::m3_params::render_stamp_coalesce(); + let internal_timestamps=crate::m3_params::render_internal_timestamps(); + let chip=self.device.soc().chip_id; + if let NativeJob::Render(j)=&mut self.jobs[index] { + let mut commands=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + for (p,next) in entries {commands.push(p.commands[*next],GFP_KERNEL).map_err(|_|early(ENOMEM))?;} + j.begin_batch(&self.drm,&self.uat,&packet.vm,&commands,base,overlap).map_err(early)?; + let prepared=(||->Result { + for (slot,(p,next)) in entries.iter().enumerate() { + let crate::m3_submit::Command::Render{command,usc}=p.commands[*next] else {return Err(EINVAL);}; + let timestamps=p.render_timestamp_addresses(*next); + let coalesce=policy::coalesce_render_stamps(coalesce_render,chip, + overlap,slot,batch_count,timestamps); + let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,chip,pair)); + j.append(command,usc,coalesce,timed)?; + j.set_user_timestamps(timestamps)?; + coalesced_stamps+=u64::from(coalesce); + } + Ok(()) + })(); + if let Err(error)=prepared { + // Running firmware can consume append's WRITE updates without a new doorbell. + // Even an append that returns an error may have changed shared counters or one + // pipe. Do not rewind live producer state or let the caller release owners and + // reuse these slots. Its published-error path stops scheduling and retains them. + if overlap {return Err((error,true));} + j.abort_batch().map_err(|e|(e,true))?; + return Err(early(error)); + } + } + let preparation_ns=prepare.elapsed().as_nanos(); + self.t8122_verdict=false; + let previous_events=self.config.completed_events; + let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; + self.state.health.set_gpu_pending(true); + self.gpu_pending = true; + let late=|e:Error|(e,true); + let expected_events=match &self.jobs[index] { + NativeJob::Compute(j)=>{ + self.config.submit_queue(2,j.queue(),j.head(),2,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000002).map_err(late)?;1 + }, + NativeJob::Render(j)=>{ + if j.first() {self.config.render_pb().map_err(late)?;} + let queues=j.queues(); + self.config.submit_queue(1,queues[1],j.heads()[1],1,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000001).map_err(late)?; + self.config.submit_queue(0,queues[0],j.heads()[0],0,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000000).map_err(late)?;2 + }, + }; + let measure=crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary); + let (base,last_draw)=match &self.jobs[index] {NativeJob::Render(j)=>{let (b,_,d)=j.batch();(b,d)},_=>(0,0)}; + Ok(Batch { entries:owned, index, kind:usize::from(matches!(control,crate::m3_submit::Command::Compute(_))), + first:control, preparation_ns, previous_events, expected_events, + start:Instant::::now(), measure, doorbell_counter_ns:if measure {physical_counter() as i64 * 1000 / 24} else {0}, + polling_ns:0, polls:0, stamp_ns:-1, first_event_ns:-1, events_ns:-1, completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0, coalesced_stamps, base, last_draw, publication_tick, mixed:None }) + } + /// Prepare both idle queues from one packet. Cross-engine changes wait on + /// the preceding command's firmware stamp; all commands remain GPU ordered. + /// Producer storage and DMA guards survive every partial publication error. + fn publish_mixed(&mut self,entries:&[(Arc,usize)], + base:usize,overlap:bool)->core::result::Result { + let early=|e:Error|(e,false); + if overlap || base!=0 || self.device.soc().chip_id!=0x6030 + || !self.render_initialized || entries.len()>policy::SLOTS {return Err(early(EINVAL));} + let packet=&entries[0].0; + if entries.iter().any(|(p,_)|!Arc::ptr_eq(packet,p)) {return Err(early(EINVAL));} + let prepare=Instant::::now(); + let internal_timestamps=crate::m3_params::render_internal_timestamps(); + self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + self.jobs.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + self.packets.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut owned=KVec::with_capacity(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut render_commands=KVec::with_capacity(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut first_compute=None; + let mut computes=0; + for (p,i) in entries { + owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + match p.commands[*i] { + c@crate::m3_submit::Command::Render{..}=>render_commands.push(c,GFP_KERNEL).map_err(|_|early(ENOMEM))?, + crate::m3_submit::Command::Compute(c)=>{computes+=1;first_compute.get_or_insert(c);}, + } + } + let renders=render_commands.len(); + if renders==0 || computes==0 {return Err(early(EINVAL));} + let index=self.jobs.iter().position(|j|matches!(j,NativeJob::Render(r) if r.started())).ok_or(early(EINVAL))?; + let existing=self.jobs.iter().position(|j|matches!(j,NativeJob::Compute(_))); + let compute_index=existing.unwrap_or(self.jobs.len()); + // Keep the packet before any producer mutation. No other batch is live. + self.packets.clear(); + self.packets.push(packet.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut render_prepared=false; + let mut compute_prepared=false; + let prepared=(||->Result { + let control=first_compute.ok_or(EINVAL)?; + if existing.is_none() { + let job=crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, + self.config.stats_region()?,control,self.device.soc().registers)?; + self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?; + } else { + let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; + j.replay(&self.uat,&packet.vm,control)?; + } + compute_prepared=true; + if let NativeJob::Compute(j)=&mut self.jobs[compute_index] { + j.reserve_barriers(&self.drm,&self.uat)?; + } + let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; + j.begin_batch(&self.drm,&self.uat,&packet.vm,&render_commands,0,false)?; + render_prepared=true; + let mut previous:Option<(bool,crate::m3_sync_layout::CompletionDependency)>=None; + let mut compute_slot=0; + let mut dependencies=0; + for (p,i) in entries { + match p.commands[*i] { + crate::m3_submit::Command::Render{command,usc}=>{ + let predecessor=previous.filter(|(render,_)|!*render).map(|(_,d)|d); + let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; + let timestamps=p.render_timestamp_addresses(*i); + let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,self.device.soc().chip_id,pair)); + j.append_after(command,usc,false,predecessor,timed)?; + j.set_user_timestamps(timestamps)?; + dependencies+=usize::from(predecessor.is_some()); + previous=Some((true,j.completion_dependency()?)); + }, + crate::m3_submit::Command::Compute(c)=>{ + let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; + if compute_slot!=0 {j.append(c)?;} + if let Some((true,dependency))=previous { + j.insert_dependency(dependency)?; + dependencies+=1; + } + // Every cross-engine predecessor flushes its stamp. + // Final queue, timestamp and cache visibility are unchanged. + j.prepare_completion(false)?; + j.set_user_timestamps(p.timestamp_addresses(*i))?; + j.set_attachments(&p.attachments[*i])?; + previous=Some((false,j.completion_dependency()?)); + compute_slot+=1; + }, + } + } + Ok(dependencies) + })(); + let dependencies=match prepared { + Ok(n)=>n, + Err(error)=>{ + if render_prepared { + let NativeJob::Render(j)=&mut self.jobs[index] else {return Err((EIO,true));}; + j.abort_batch().map_err(|e|(e,true))?; + } + if compute_prepared { + if existing.is_none() {drop(self.jobs.remove(compute_index).map_err(|_|(EIO,true))?);} + else if let NativeJob::Compute(j)=&mut self.jobs[compute_index] { + j.abort_batch().map_err(|e|(e,true))?; + } + } + return Err(early(error)); + }, + }; + let preparation_ns=prepare.elapsed().as_nanos(); + let previous_events=self.config.completed_events; + let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; + self.state.health.set_gpu_pending(true);self.gpu_pending=true; + let late=|e:Error|(e,true); + let NativeJob::Render(j)=&self.jobs[index] else {return Err(late(EIO));}; + let queues=j.queues();let heads=j.heads();let (_,_,last_draw)=j.batch(); + self.config.submit_queue(1,queues[1],heads[1],1,false).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000001).map_err(late)?; + self.config.submit_queue(0,queues[0],heads[0],0,false).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000000).map_err(late)?; + let NativeJob::Compute(j)=&self.jobs[compute_index] else {return Err(late(EIO));}; + self.config.submit_queue(2,j.queue(),j.head(),2,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000002).map_err(late)?; + Ok(Batch {entries:owned,index,kind:2,first:packet.commands[entries[0].1], + preparation_ns,previous_events,expected_events:3,start:Instant::::now(), + measure:crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary), + doorbell_counter_ns:0,polling_ns:0,polls:0,stamp_ns:-1,first_event_ns:-1, + events_ns:-1,completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0,coalesced_stamps:0,base:0,last_draw,publication_tick, + mixed:Some(MixedBatch {compute_index,renders,computes,dependencies})}) + } + /// A mixed publication retires only after both jobs, all three queue + /// messages, firmware health and engine idle passed the ordinary checks. + /// Its wall time is reported once rather than charged to both engines. + fn retire_mixed(&mut self,batch:&Batch,mixed:&MixedBatch)->Result { + let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; + let render_ns=j.gpu_ns(0,mixed.renders)?; + let stages=j.stage_ns(0,mixed.renders)?; + j.progress(&self.drm)?; + let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; + let compute_ns=j.gpu_ns()?; + j.progress(&self.drm)?; + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let vm=batch.entries[0].0.vm.id(); + let observed_tick=physical_counter(); + let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; + let render_span=j.gpu_ticks(0,mixed.renders)?; + let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; + let compute_span=j.batch_gpu_span()?; + let start=render_span.0.min(compute_span[0]); + let end=render_span.1.max(compute_span[1]); + let valid=batch.publication_tick!=0 && start>=batch.publication_tick && end>=start && observed_tick>=end; + dev_info!(self.drm.as_ref(),"M3_BATCH_BOUNDARY vm={} mixed=1 batch={} publication_tick={} gpu_start_tick={} gpu_end_tick={} observed_tick={} counter_hz=24000000 valid={}\n", + vm,batch.entries.len(),batch.publication_tick,start,end,observed_tick,u32::from(valid)); + self.log_batch_milestones(batch,render_span.0,render_span.1,render_ns,stages[1]); + // Render and compute have separate slot sequences in one mixed batch. + let mut render_slot=0; + let mut compute_slot=0; + for (packet,next) in batch.entries.iter() { + match packet.commands[*next] { + crate::m3_submit::Command::Render{command:r,..}=>{ + let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; + let s=j.slot_stage_ns(0,mixed.renders,render_slot)?; + dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} mixed=1 slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n", + vm,render_slot,mixed.renders,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); + render_slot+=1; + }, + crate::m3_submit::Command::Compute(c)=>{ + let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; + let ns=j.slot_gpu_ns(compute_slot)?; + dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} mixed=1 slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n", + vm,compute_slot,mixed.computes,c.base,c.end,c.usc_base,ns); + compute_slot+=1; + }, + } + } + } + self.state.health.record_completion(); + self.render_batches[mixed.renders-1]+=1; + self.pass_cost.record(batch.entries[0].0.vm.id(),render_ns/mixed.renders as u64); + self.mixed_batches+=1; + if (self.mixed_batches==1 || self.mixed_batches%128==0) && batch.measure { + dev_info!(self.drm.as_ref(),"M3_MIXED_BATCH_RETIRED count={} commands={} render={} compute={} barriers={} prepare_ns={} active_ns={} render_gpu_ns={} compute_gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} stamps=1 queues=1 events=3 idle=1\n", + self.mixed_batches,batch.entries.len(),mixed.renders,mixed.computes,mixed.dependencies, + batch.preparation_ns,batch.start.elapsed().as_nanos(),render_ns,compute_ns,stages[0],stages[1],stages[2]); + } + if self.active.is_empty() { + self.gpu_pending=false;self.state.health.set_gpu_pending(false); + } + Ok(()) + } + /// Check the active batch once. Ok(true): retired (its packets are signalled). + fn poll_active(&mut self)->Result { + let Some((index,measure,last_draw))=self.active.first().map(|b|(b.index,b.measure,b.last_draw)) else {return Ok(false);}; + let poll_start=measure.then(Instant::::now); + if !self.state.healthy() || self.config.drain(&self.drm).is_err() { + self.fail_active(EIO);return Err(EIO); + } + // With later batches in flight, the oldest retires once its own last draw's stamps + // have arrived; the last one is also checked for exact queue, event and engine idle. + if self.active.len()>1 { + let done=match &mut self.jobs[index] {NativeJob::Render(j)=>j.complete_through(last_draw),_=>Err(EINVAL)}; + match done { + Ok(true)=>{ + let batch=self.active.remove(0).map_err(|_|EIO)?; + if let Err(e)=self.retire(&batch) { + self.capture_fault(e);Self::finish_all(&batch.entries,Err(e)); + self.fail_active(e);return Err(e); + } + self.retire_packets(&batch.entries); + return Ok(true); + } + Ok(false)=>{ + let (expired,previous)=self.active.first().map(|b|(b.start.elapsed()>=Delta::from_secs(2),b.previous_events)).ok_or(EIO)?; + if expired { + dev_err!(self.drm.as_ref(),"M3 pipelined completion events={} previous={}\n",self.config.completed_events,previous); + self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT); + } + return Ok(false); + } + Err(e)=>{self.fail_active(e);return Err(e);} + } + } + let secondary=self.active.first().and_then(|b|b.mixed.as_ref().map(|m|m.compute_index)); + let completion=(||->Result { + let primary=self.jobs[index].complete()?; + let other=match secondary {Some(i)=>self.jobs[i].complete()?,None=>true}; + Ok(primary && other) + })(); + let stamped=match completion {Ok(v)=>v,Err(e)=>{self.fail_active(e);return Err(e);}}; + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let before=physical_counter(); + let parts=match &mut self.jobs[index] { + NativeJob::Render(j)=>j.fragment_stamp_parts().ok(), + _=>None, + }; + let after=physical_counter(); + if let Some(parts)=parts { + if let Some(batch)=self.active.first_mut() { + if batch.fragment_host_tick==0 { + if parts[0] {batch.fragment_host_tick=after;} + else {batch.fragment_host_pending_tick=before;} + } + if batch.fragment_fw_tick==0 { + if parts[1] {batch.fragment_fw_tick=after;} + else {batch.fragment_fw_pending_tick=before;} + } + } + } + } + let power_enabled=crate::m3_params::active_power(); + if power_enabled { + let since_last=self.active_power_last.map(|last|last.elapsed().as_nanos()); + if crate::m3_active_power::due(power_enabled,self.device.soc().chip_id, + stamped,self.active_power_samples,since_last) { + self.active_power_last=Some(Instant::::now()); + self.active_power_samples+=1; + // Diagnostic read errors do not change job completion or recovery. + if let Err(e)=self.config.log_active_power(&self.drm,&self.device, + self.active_power_samples) { + dev_info!(self.drm.as_ref(),"M3_ACTIVE_POWER sample={} unavailable={:?}\n", + self.active_power_samples,e); + } + } + } + let completion_parts=if measure && secondary.is_none() { + match &mut self.jobs[index] { + NativeJob::Render(j)=>Some(if stamped {[true;4]} else { + match j.completion_parts() {Ok(v)=>v,Err(e)=>{self.fail_active(e);return Err(e);}} + }), + _=>None, + } + } else {None}; + let completed=self.config.completed_events; + let (evented,expired,previous)={ + let batch=self.active.first_mut().ok_or(EIO)?; + let evented=completed>=batch.previous_events+batch.expected_events; + if batch.measure { + let at=batch.start.elapsed().as_nanos(); + if stamped && batch.stamp_ns<0 {batch.stamp_ns=at;} + if let Some(parts)=completion_parts { + for (part,done) in parts.iter().enumerate() { + if *done && batch.completion_ns[part]<0 {batch.completion_ns[part]=at;} + } + } + if completed>batch.previous_events && batch.first_event_ns<0 {batch.first_event_ns=at;} + if evented && batch.events_ns<0 {batch.events_ns=at;} + } + if let Some(t)=poll_start {batch.polling_ns+=t.elapsed().as_nanos();batch.polls+=1;} + (evented,batch.start.elapsed()>=Delta::from_secs(2),batch.previous_events) + }; + if stamped && evented { + let retire_mmio=crate::m3_params::retire_mmio(self.device.soc()); + match self.device.check_idle_parts( + retire_mmio&crate::m3_params::RETIRE_MMIO_BUSY!=0, + retire_mmio&crate::m3_params::RETIRE_MMIO_FAULTS!=0) { + Ok(())=>match self.config.pipes_idle() { + Ok(true)=>{ + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let tick=physical_counter(); + if let Some(batch)=self.active.first_mut() {batch.pipes_idle_tick=tick;} + } + if retire_mmio&crate::m3_params::RETIRE_MMIO_PSTATE!=0 { + if let Err(e)=self.config.check_pstate(&self.drm,&self.device,"after a job") { + if crate::t8122_start::is_t8122(self.device.soc()) { + crate::t8122_start::cap_violated_verdict(self.drm.as_ref(),e); + self.t8122_verdict=true; + } else if crate::t6031_start::is_t6031(self.device.soc()) { + crate::t6031_start::cap_violated_verdict(self.drm.as_ref(), e); + self.t8122_verdict=true; + } + self.fail_active(e);return Err(e); + } + } + // Stamps, both queue indices, required events, + // firmware health, engines and pipes are verified. + let batch=self.active.remove(0).map_err(|_|EIO)?; + if let Err(e)=self.retire(&batch) { + self.capture_fault(e); + Self::finish_all(&batch.entries,Err(e)); + self.fail_pending(e); + return Err(e); + } + // Signal every packet whose last command was in this batch. A packet + // that continues with other commands is still at the front of `pending`. + self.retire_packets(&batch.entries); + return Ok(true); + }, + Ok(false)=>{}, // Firmware must consume every submitted queue message. + Err(e)=>{self.fail_active(e);return Err(e);} + }, + Err(e) if e==EBUSY=>{}, // Retirement can trail the event. + Err(e)=>{self.fail_active(e);return Err(e);} + } + } + if expired { + dev_err!(self.drm.as_ref(),"M3 completion events={} previous={}\n",completed,previous); + self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT); + } + Ok(false) + } + fn log_batch_milestones(&self,batch:&Batch,render_start:u64,render_end:u64, + render_gpu_ns:u64,fragment_sum_ns:u64) { + dev_info!(self.drm.as_ref(),"M3_BATCH_MILESTONE vm={} publication_tick={} draw={} mixed={} render_start_tick={} render_end_tick={} render_gpu_ns={} fragment_sum_ns={} fragment_host_tick={} fragment_fw_tick={} fragment_host_pending_tick={} fragment_fw_pending_tick={} pipes_idle_tick={} counter_hz=24000000\n", + batch.entries[0].0.vm.id(),batch.publication_tick,batch.last_draw,u32::from(batch.mixed.is_some()), + render_start,render_end,render_gpu_ns,fragment_sum_ns,batch.fragment_host_tick,batch.fragment_fw_tick, + batch.fragment_host_pending_tick,batch.fragment_fw_pending_tick,batch.pipes_idle_tick); + for (packet,command) in batch.entries.iter() { + dev_info!(self.drm.as_ref(),"M3_BATCH_PACKET vm={} publication_tick={} packet={:#x} command={}\n", + packet.vm.id(),batch.publication_tick,packet.profile_id(),command); + } + } + /// Bookkeeping for a retired batch, then signal its completed packets. + fn retire(&mut self,batch:&Batch)->Result { + if let Some(mixed)=&batch.mixed {return self.retire_mixed(batch,mixed);} + let retire_start=batch.measure.then(Instant::::now); + let index=batch.index;let batch_count=batch.entries.len();let control=batch.first; + self.state.health.record_completion(); + if self.active.is_empty() { + match &mut self.jobs[index] { + NativeJob::Compute(j)=>j.progress(&self.drm)?, + NativeJob::Render(j)=>j.progress(&self.drm)?, + } + } + let retired_counter_ns=if batch.measure {physical_counter() as i64 * 1000 / 24} else {0}; + let ticks=match &mut self.jobs[index] { + NativeJob::Render(j) if batch.measure=>Some(j.gpu_ticks(batch.base,batch_count)?), + _=>None, + }; + let stages=match &mut self.jobs[index] { + NativeJob::Render(j)=>j.stage_ns(batch.base,batch_count)?, + _=>[0;3], + }; + let (kind,gpu_ns)=match &mut self.jobs[index] { + NativeJob::Compute(j)=>(1,j.gpu_ns()?), + NativeJob::Render(j)=>(0,j.gpu_ns(batch.base,batch_count)?), + }; + if kind==0 { + self.render_batches[batch_count-1]+=1; + if batch.measure && batch.coalesced_stamps>0 && self.coalesced_render_stamps==0 { + if let NativeJob::Render(j)=&mut self.jobs[index] { + let masks=j.stamp_flush_masks(batch.base,batch_count)?; + dev_info!(self.drm.as_ref(),"M3_RENDER_STAMP_FLUSH vm={} batch={} ta_mask={:#x} fs_mask={:#x} coalesced={}\n", + batch.entries[0].0.vm.id(),batch_count,masks[0],masks[1],batch.coalesced_stamps); + } + } + self.coalesced_render_stamps+=batch.coalesced_stamps; + let vm=batch.entries[0].0.vm.id(); + self.pass_cost.record(vm,gpu_ns/batch_count as u64); + self.render_initialized=true; + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let observed_tick=physical_counter(); + let (cs,span)=match (&mut self.jobs[index],control) { + (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>(c.base,j.batch_gpu_span()?), + (NativeJob::Render(j),crate::m3_submit::Command::Render{command:r,..})=>{let (a,b)=j.gpu_ticks(batch.base,batch_count)?;(r.vdm_base,[a,b])}, + _=>return Err(EINVAL), + }; + let valid=batch.publication_tick!=0 && span[0]>=batch.publication_tick && span[1]>=span[0] && observed_tick>=span[1]; + dev_info!(self.drm.as_ref(),"M3_BATCH_BOUNDARY vm={} cs={:#x} batch={} publication_tick={} gpu_start_tick={} gpu_end_tick={} observed_tick={} counter_hz=24000000 valid={}\n", + batch.entries[0].0.vm.id(),cs,batch_count,batch.publication_tick,span[0],span[1],observed_tick,u32::from(valid)); + self.log_batch_milestones(batch, + if kind==0 {span[0]} else {0},if kind==0 {span[1]} else {0}, + if kind==0 {gpu_ns} else {0},stages[1]); + if let NativeJob::Compute(j)=&mut self.jobs[index] { + for (slot,(p,next)) in batch.entries.iter().enumerate() { + let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + let ns=j.slot_gpu_ns(slot)?; + dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n",p.vm.id(),slot,batch_count,c.base,c.end,c.usc_base,ns); + } + } + if let NativeJob::Render(j)=&mut self.jobs[index] { + for (slot,(p,next)) in batch.entries.iter().enumerate() { + let crate::m3_submit::Command::Render{command:r,..}=p.commands[*next] else {return Err(EINVAL);}; + let s=j.slot_stage_ns(batch.base,batch_count,slot)?; + dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n",p.vm.id(),slot,batch_count,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); + } + } + } + let preparation_ns=batch.preparation_ns; + let active_ns=batch.start.elapsed().as_nanos(); + self.state.health.timing.record(kind==0, batch_count as u64, + preparation_ns, active_ns, gpu_ns, stages); + let retirement_ns=retire_start.map_or(0,|t|t.elapsed().as_nanos()); + if batch.measure { + if let crate::m3_submit::Command::Render {command:r,..}=control { + if batch_count>1 {self.geometry_overflow+=1;} else { + if let Some(g)=self.geometry.iter_mut().find(|g| + g.count==0 || (g.width==r.width && g.height==r.height && g.samples==r.samples)) { + g.width=r.width;g.height=r.height;g.samples=r.samples; + g.count+=1;g.prepare_ns+=preparation_ns;g.active_ns+=active_ns; + g.gpu_ns+=gpu_ns;g.tiling_ns+=stages[0];g.fragment_ns+=stages[1]; + g.max_fragment_ns=g.max_fragment_ns.max(stages[1]); + g.polling_ns+=batch.polling_ns;g.retirement_ns+=retirement_ns; + g.polls+=batch.polls; + g.stamp_ns+=batch.stamp_ns;g.first_event_ns+=batch.first_event_ns;g.events_ns+=batch.events_ns; + for (total,observed) in g.completion_ns.iter_mut().zip(batch.completion_ns) {*total+=observed;} + if let Some((ta,fe))=ticks { + g.pickup_ns+=(ta as i64)*1000/24-batch.doorbell_counter_ns; + g.tail_ns+=retired_counter_ns-(fe as i64)*1000/24; + } + } else { + self.geometry_overflow+=1; + } + } + } + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + match control { + crate::m3_submit::Command::Render {command:r,..}=> + dev_info!(self.drm.as_ref(),"M3_PASS kind=render batch={} vdm={:#x} width={} height={} samples={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={}\n",batch_count,r.vdm_base,r.width,r.height,r.samples,preparation_ns,active_ns,gpu_ns,stages[0],stages[1],stages[2]), + crate::m3_submit::Command::Compute(_)=> + dev_info!(self.drm.as_ref(),"M3_PASS kind=compute prepare_ns={} active_ns={} gpu_ns={}\n",preparation_ns,active_ns,gpu_ns), + } + } + let t=&mut self.timing[kind]; + t[0]+=1;t[1]+=preparation_ns;t[2]+=active_ns;t[3]+=gpu_ns as i64; + for i in 0..3 {t[4+i]+=stages[i] as i64;} + // Running totals are in debugfs `timing`; the periodic line is a SubmitTiming diagnostic. + if t[0]==1 && crate::t8122_start::is_t8122(self.device.soc()) { + let span=match &mut self.jobs[index] { + NativeJob::Compute(j)=>j.batch_gpu_span().ok(), + NativeJob::Render(j)=>j.gpu_ticks(batch.base,batch_count).ok().map(|(a,b)|[a,b]), + }; + crate::t8122_start::job_completed_verdict(self.drm.as_ref(),kind,gpu_ns,span); + } else if t[0]==1 && crate::t6031_start::is_t6031(self.device.soc()) { + crate::t6031_start::job_completed_verdict(self.drm.as_ref(), kind, gpu_ns); + } + if t[0]%128==0 && crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + dev_info!(self.drm.as_ref(),"M3_TIMING kind={} count={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} ordinal={}\n",kind,t[0],t[1],t[2],t[3],t[4],t[5],t[6], + match &self.jobs[index] {NativeJob::Render(j)=>j.ordinal(),NativeJob::Compute(j)=>j.ordinal()}); + } + if kind==0 && t[0]%512==0 && + crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary) { + let early=match &self.jobs[index] {NativeJob::Render(j)=>j.early_count(),_=>0}; + dev_info!(self.drm.as_ref(),"M3_ORDERED_BATCHES depths={:?} early={} packets={} coalesced_stamps={}\n", self.render_batches,early,self.coalesced_packets,self.coalesced_render_stamps); + let now=::ktime_get(); + dev_info!(self.drm.as_ref(),"M3_GEOMETRY_BATCH ns={} render_count={} overflow={}\n", + now,t[0],self.geometry_overflow); + self.geometry_overflow=0; + for g in &mut self.geometry { + if g.count==0 {continue;} + dev_info!(self.drm.as_ref(),"M3_GEOMETRY ns={} width={} height={} samples={} count={} prepare_ns={} active_ns={} gpu_ns={} tiling_ns={} fragment_ns={} max_fragment_ns={} polling_ns={} retirement_ns={} sleeping_ns={} polls={} stamp_ns={} first_event_ns={} events_ns={} pickup_ns={} tail_ns={} gpu_stamp_ns={} fw_stamp_ns={} queue_done_ns={} queue_read_ns={}\n", + now,g.width,g.height,g.samples,g.count,g.prepare_ns,g.active_ns,g.gpu_ns,g.tiling_ns,g.fragment_ns,g.max_fragment_ns,g.polling_ns,g.retirement_ns,g.sleeping_ns,g.polls,g.stamp_ns,g.first_event_ns,g.events_ns,g.pickup_ns,g.tail_ns,g.completion_ns[0],g.completion_ns[1],g.completion_ns[2],g.completion_ns[3]); + *g=GeometryTiming::default(); + } + } + if self.active.is_empty() { + self.gpu_pending = false; + self.state.health.set_gpu_pending(false); + } + for pair in batch.entries.windows(2) { + if !Arc::ptr_eq(&pair[0].0,&pair[1].0) {self.coalesced_packets+=1;} + } + Ok(()) + } +} +// Keep the large graph on the heap: returning and moving it through probe, +// Registration and Mutex initialization otherwise duplicates it on the bounded +// kernel stack. ManuallyDrop retains the whole allocation on failed ASC stop. +pub(crate) struct Runtime { inner: ManuallyDrop> } +impl Runtime { + #[inline(never)] + pub(crate) fn new(pdev: &platform::Device, device: Device, contents: crate::m3_adt_config::Contents) -> Result { + // On an armed T8122, name a setup failure before the GPU coprocessor is started (RTKit + // state, mailbox transport, the DRM device). Captured as a Copy bool so the closure does + // not borrow `device`, which is moved into Inner below. + let t8122 = crate::t8122_start::is_t8122(device.soc()); + let t6031 = crate::t6031_start::is_t6031(device.soc()); + let coproc_refused = |e: &Error| { + if t8122 { crate::t8122_start::coproc_setup_refused(pdev.as_ref(), *e); } + if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), *e); } + }; + device.require_stopped(pdev).inspect_err(coproc_refused)?; + // Reserve before starting ASC. ENOMEM must not drop firmware owners + // while the coprocessor may still access them. Writing the completed + // graph into this allocation is infallible. + let owner = KBox::::new_uninit(GFP_KERNEL)?; + let drm: driver::AsahiDevRef = kernel::drm::Device::new(pdev.as_ref(), driver::AsahiData::new(pdev, None, true)).inspect_err(coproc_refused)?; + let state = m3_rtkit::State::new(pdev, drm.clone(), device.firmware().resources.regions[5]).inspect_err(coproc_refused)?; + let mut transport = rtkit::RtKit::new(pdev.as_ref(), None, 0, state.clone()).inspect_err(coproc_refused)?; + if crate::m3_adt_config::stop_before_asc(pdev.as_ref()) { + if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), ENODEV); } + return Err(ENODEV); + } + // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. + let mut started=false; + let prepared=(|| -> Result<_> { + device.start_asc(pdev)?; + Pin::new(&mut transport).wake()?; + if !state.healthy() || !Pin::new(&transport).is_running() {return Err(EIO);} + for ep in [0x20,0x21] { + if !Pin::new(&mut transport).has_endpoint(ep) {return Err(ENODEV);} + Pin::new(&mut transport).start_endpoint(ep)?; + } + started=true; + // The UAT geometry of the admitted SoC (complete: `Soc::require_complete`). + // SAFETY: the admitted RTKit (J514S on T6030) is awake; no initdata or GPU + // job has been published. Its running PPL handoff owns the peer lock. + let uat=unsafe {mmu::Uat::new_m3_running(&drm,device.soc())}?; + let config=Config::new(&drm,&uat,device.firmware(),&contents)?; + Ok((uat,config)) + })(); + let (uat,config)=match prepared { + Ok(v)=>v, + Err(e)=>{ + crate::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); + if t6031 { crate::t6031_start::prepare_verdict(pdev.as_ref(), started, e); } + state.health.mark_failed(); + if device.stop_asc().is_err() { + unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; + core::mem::forget(transport);core::mem::forget(state); + core::mem::forget(drm);core::mem::forget(device); + } + return Err(e); + } + }; + Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, drm, device, + jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3_pass_layout::SLOTS], + geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false, + boot_step:crate::t8122_start::BootStep::Publish })) }) + } + pub(crate) fn drm(&self) -> driver::AsahiDevRef { self.inner.drm.clone() } + pub(crate) fn health(&self) -> Arc { self.inner.state.health.clone() } + pub(crate) fn new_vm(&mut self, id: u64, range: core::ops::Range) -> Result { + use crate::util::RangeExt; + let reserved_range=0x70_0000_0000..0x80_0000_0000; + if range.overlaps(reserved_range.clone()) {return Err(EINVAL);} + if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } + let vm=self.inner.uat.new_vm(id, range)?; + let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?; + vm.install_driver_mappings(KVec::new(),reserved)?; + Ok(vm) + } + pub(crate) fn map_timestamp(&self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { + if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } + bo.map_range_into_range(self.inner.uat.kernel_vm(),range,crate::agx_memory::TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64,mmu::PROT_FW_SHARED_RW,false) + } + /// Queue a packet and return its fence at once. The scheduler can hand over further + /// ready jobs up to its credit limit while the GPU runs. Only the completion worker + /// publishes: it coalesces render commands of one VM and urgency, and may keep bounded + /// same-VM batches in flight after InitBM retirement. VM/engine changes wait for full + /// retirement. A burst can therefore share a firmware round trip across packets. + pub(crate) fn submit(shared:&crate::m3_drm::Shared,packet:Arc)->Result { + crate::debug::update_debug_flags(); + let mut guard=shared.lock(); + let inner=&mut *Option::as_mut(&mut *guard).ok_or(ENODEV)?.inner; + if !inner.state.healthy() {return Err(EIO);} + if packet.commands.is_empty() || packet.commands.len()>256 { + pr_err!("M3 submit limit: commands={} maximum=256\n",packet.commands.len()); + return Err(E2BIG); + } + // Higher urgency first, FIFO within one urgency, including unfinished packets. + // More urgent work can overtake a continuation between retired batches. + // Packets of one queue share an urgency, so their order is kept. + let urgency=packet.urgency; + let at=policy::pending_position(inner.pending.iter().map(|(p,_)|p.urgency),urgency); + if inner.pending.len()>=policy::MAX_PACKETS {return Err(EBUSY);} + inner.pending.reserve(1,GFP_KERNEL)?; + inner.pending.insert_within_capacity(at,(packet,0)).map_err(|_|ENOMEM)?; + let drm=inner.drm.clone(); + drop(guard); + crate::m3_completion::queue(&drm); + Ok(()) + } + /// Withdraw a packet's unpublished remainder on cancellation or timeout. Its active + /// commands remain owned until retirement; error signaling cannot release a DMA guard. + pub(crate) fn withdraw(shared:&crate::m3_drm::Shared,packet:&Arc)->bool { + let mut guard=shared.lock(); + let Some(runtime)=Option::as_mut(&mut *guard) else {return false;}; + let inner: &mut Inner=&mut runtime.inner; + let removed=inner.pending.iter().position(|(p,_)|Arc::ptr_eq(p,packet)) + .is_some_and(|i|inner.pending.remove(i).is_ok()); + if !inner.packet_active(packet) && !inner.fault_captured {packet.release_guard();} + removed + } + pub(crate) fn completion_wait(&self) -> Arc { self.inner.state.events.clone() } + pub(crate) fn events(&self)->Arc {self.inner.state.events.clone()} + /// One completion-worker step: retire the active batch if it has completed, then publish + /// the next one. The worker drops the runtime lock between steps. + pub(crate) fn service_job(&mut self)->Service { + let chip=self.inner.device.soc().chip_id; + let busy=self.inner.state.healthy() && + (!self.inner.active.is_empty() || !self.inner.pending.is_empty()); + if let Err(error)=self.inner.config.performance_request(chip,busy) { + self.inner.fail_active(error); + return Service::Idle; + } + let result=self.service_job_inner(); + if matches!(result,Service::Idle) || !self.inner.state.healthy() || + (self.inner.active.is_empty() && self.inner.pending.is_empty()) { + let _=self.inner.config.performance_request(chip,false); + } + result + } + fn service_job_inner(&mut self)->Service { + let inner: &mut Inner=&mut *self.inner; + if inner.active.is_empty() { + if inner.fault_captured || !inner.state.healthy() {inner.fail_pending(EIO);return Service::Idle;} + if let Err(error)=inner.config.drain(&inner.drm) { + inner.capture_fault(error);inner.fail_pending(EIO);return Service::Idle; + } + return if inner.publish_next() {Service::Progress} else {Service::Idle}; + } + let messages=inner.state.event_messages.load(Ordering::Acquire); + match inner.poll_active() { + Ok(true)=>{inner.publish_next();Service::Progress}, + Ok(false)=>if inner.publish_next() {Service::Progress} else {Service::Waiting(messages)}, + Err(_)=>{inner.fail_pending(EIO);Service::Idle}, + } + } + pub(crate) fn boot(&mut self, pdev: &platform::Device) -> Result { + let is_t8122 = crate::t8122_start::is_t8122(self.inner.device.soc()); + let is_t6031 = crate::t6031_start::is_t6031(self.inner.device.soc()); + self.inner.boot_step = crate::t8122_start::BootStep::Publish; + let result = self.boot_inner(pdev); + if let Err(error) = result { + // On T8122, log the firmware readiness words on every boot failure, so the verdict's + // "see M3 firmware readiness" always has a line to point at. The ready-wait timeout in + // boot_inner already logs them (on every SoC, as in 12.0), so skip that case here. + let logged = self.inner.boot_step == crate::t8122_start::BootStep::AwaitReady + && error == ETIMEDOUT; + if (is_t8122 || is_t6031) && !logged { + let inner: &mut Inner = &mut *self.inner; + let _ = inner.config.log_ready(&inner.drm); + } + self.inner.capture_fault(error); + } + if is_t8122 { + let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); + crate::t8122_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, + self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); + } + if is_t6031 { + let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); + crate::t6031_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, + self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); + } + result + } + fn boot_inner(&mut self, pdev: &platform::Device) -> Result { + use crate::t8122_start::BootStep; + let root = self.inner.config.root(); + dev_info!(pdev.as_ref(), "M3: publishing owned initdata {:#x}\n", root); + self.inner.boot_step = BootStep::Publish; + Pin::new(&mut self.inner.transport).send_message(0x20, 0x0081000000000000 | (root & ((1u64<<44)-1)))?; + self.inner.boot_step = BootStep::DeviceControl; + self.send_control(0x13)?; + self.send_control(9)?; + self.inner.boot_step = BootStep::AwaitReady; + let start = Instant::::now(); + while !self.inner.config.ready()? { + if start.elapsed() >= Delta::from_secs(2) || !self.inner.state.healthy() { + let inner: &mut Inner=&mut *self.inner;let _=inner.config.log_ready(&inner.drm); + return Err(ETIMEDOUT); + } + fsleep(Delta::from_millis(1)); + } + self.inner.boot_step = BootStep::PostReady; + // Import the configured idle policy again after initial power-up. + self.send_control(0x13)?; + let inner: &mut Inner = &mut *self.inner; + inner.config.drain(&inner.drm)?; + inner.config.log_ready(&inner.drm)?; + if inner.device.soc().chip_id==0x6030 && + (crate::m3_params::active_power() || crate::m3_params::globals_debug_off()) { + inner.config.log_globals_debug(&inner.drm)?; + } + inner.config.check_pstate(&inner.drm, &inner.device, "after boot")?; + inner.state.health.set_gpu_pending(false); + dev_info!(pdev.as_ref(), "M3: firmware accepted owned initdata and device controls\n"); + Ok(()) + } + fn send_control(&mut self, opcode: u32) -> Result { + let next = match self.inner.config.control(opcode) { + Ok(next) => next, + Err(error) => { + dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} queue failed ({:?})\n", opcode, error); + return Err(error); + } + }; + let send_start = Instant::::now(); + if let Err(error) = Pin::new(&mut self.inner.transport).send_message(0x21, 0x0083000000000011) { + dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} mailbox send failed ({:?}), elapsed_ns={}\n", + opcode, error, send_start.elapsed().as_nanos()); + return Err(error); + } + let start = Instant::::now(); + loop { + let inner: &mut Inner = &mut *self.inner; + inner.config.drain(&inner.drm)?; + if !inner.state.healthy() { return Err(EIO); } + if inner.config.control_done(next)? { return Ok(()); } + if start.elapsed() >= Delta::from_secs(2) { + dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} acknowledgement timed out, elapsed_ns={}\n", + opcode, start.elapsed().as_nanos()); + return Err(ETIMEDOUT); + } + fsleep(Delta::from_millis(1)); + } + } +} +impl Drop for Runtime { + fn drop(&mut self) { + let chip=self.inner.device.soc().chip_id; + let _=self.inner.config.performance_request(chip,false); + self.inner.state.health.mark_failed(); + // Unpublished packets never reach the firmware; a published batch's packets stay + // retained below if the GPU may still use them, but their fences must not hang. + self.inner.fail_pending(ENODEV); + for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} + if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { + // SAFETY: a live module reference pins callbacks with retained DMA. + unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; + return; + } + // SAFETY: no GPU command is pending and ASC is stopped. Field order + // drains callbacks before dropping retained commands, UAT and power. + unsafe { ManuallyDrop::drop(&mut self.inner) }; + } +} + +/// Same scaling contract as G16's qualified publication profiler. AP counter +/// rates may differ between SoCs; firmware command timestamps use 24 MHz. +pub(crate) fn physical_counter() -> u64 { + let tick: u64; + let frequency: u64; + // SAFETY: Read architectural counters only; no device state is modified. + unsafe { core::arch::asm!("mrs {f}, cntfrq_el0", "mrs {t}, cntpct_el0", + f = out(reg) frequency, t = out(reg) tick, + options(nomem, nostack, preserves_flags)) }; + let frequency = frequency & 0xffff_ffff; + if frequency == 0 { return 0; } + (tick / frequency) * 24_000_000 + (tick % frequency) * 24_000_000 / frequency +} diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs new file mode 100644 index 0000000000000..9bd08d0257b0c --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded M3 compute and graphics submissions through the common DRM scheduler/syncobjs. +use kernel::{c_str,dma_fence::*,drm::sched,prelude::*,sync::{Arc,Mutex},new_mutex,xarray}; +use crate::{file,agx_uapi,mmu,queue,m3_drm::Shared}; +use core::sync::atomic::{AtomicBool, Ordering}; +#[derive(Default)] +pub(crate) struct Completion; +#[vtable] +impl FenceOps for Completion { + fn get_driver_name<'a>(self:&'a FenceObject)->&'a CStr {c_str!("asahi")} + fn get_timeline_name<'a>(self:&'a FenceObject)->&'a CStr {c_str!("m3-native")} +} +pub(crate) struct Destination { mapping: Arc, offset: usize } +impl Destination { + fn resolve(objects: Pin<&xarray::XArray>>, + timestamp: agx_uapi::UapiTimestamp) -> Result> { + if timestamp.handle == 0 { return Ok(None); } + let guard = objects.lock(); + let object = guard.get(timestamp.handle.try_into()?).ok_or(ENOENT)?; + match &*object { + file::Object::TimestampBuffer(mapping) => { + let offset = timestamp.offset as usize; + if offset & 7 != 0 || offset.checked_add(8).ok_or(EOVERFLOW)? > mapping.size() { + return Err(EINVAL); + } + Ok(Some(Self { mapping: mapping.clone(), offset })) + } + } + } + fn firmware_address(&self) -> u64 { self.mapping.iova() + self.offset as u64 } +} + +#[derive(Copy,Clone)] +pub(crate) enum Command { + Compute(crate::agx_compute::Control), + Render{command:agx_uapi::UapiRenderCommand,usc:u64}, +} +pub(crate) struct Packet { + pub(crate) vm:mmu::Vm, + pub(crate) commands:KVec, + timestamps:KVec<[Option;4]>, + pub(crate) attachments:KVec, + pub(crate) wide_visibility:KVec, + completion:UserFence, + finish_claimed: AtomicBool, + /// Queue urgency (3 = UAPI realtime .. 0 = UAPI low): more urgent packets are published first. + pub(crate) urgency:u8, + /// Who submitted the packet, for the failure log only. + pub(crate) owner:Owner, + vm_job:Pin>>>, +} +/// The process and thread that submitted a packet, recorded in the submit ioctl. Only logged: +/// a renamed task leaves a stale name, which is harmless. +#[derive(Copy,Clone)] +pub(crate) struct Owner {pub(crate) pid:i32,process:[u8;16],thread:[u8;16]} +impl Owner { + fn current()->Self { + let task=kernel::current!(); + let leader=task.group_leader(); + Self {pid:leader.pid(),process:crate::m3_client::task_comm(leader), + thread:crate::m3_client::task_comm(task)} + } + fn name(comm:&[u8;16])->&str { + let len=comm.iter().position(|&b|b==0).unwrap_or(comm.len()); + core::str::from_utf8(&comm[..len]).unwrap_or("?") + } + pub(crate) fn process(&self)->&str {Self::name(&self.process)} + pub(crate) fn thread(&self)->&str {Self::name(&self.thread)} +} +impl Packet { + pub(crate) fn profile_id(&self)->usize {self as *const Self as usize} + pub(crate) fn timestamp_addresses(&self,index:usize)->[u64;2] { + core::array::from_fn(|i|self.timestamps[index][i].as_ref().map_or(0,Destination::firmware_address)) + } + pub(crate) fn render_timestamp_addresses(&self,index:usize)->[[u64;2];2] { + core::array::from_fn(|stage| core::array::from_fn(|i| + self.timestamps[index][stage*2+i].as_ref().map_or(0,Destination::firmware_address))) + } + /// Called only after all published commands retired or before any publication. + pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} + pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} + pub(crate) fn finish(&self,result:Result) { + // Retirement, cancellation and a queued scheduler timeout can race. + // Claim error/signal ownership once; this flag is NOT proof that the + // fence is signaled or that hardware DMA has stopped. + if self.finish_claimed.swap(true, Ordering::Relaxed) { return; } + if let Err(e)=result {if let Ok(status) = self.vm.agx_status() { status.record(e.to_errno()); }self.completion.set_error(e);} + // A scheduler job/fence and the runtime's last packet can outlive + // hardware retirement. They must not keep the VM's active-job count + // nonzero: GEM closes otherwise accumulate across completed frames. + // On failure the runtime quarantines this guard with the packet until + // firmware has provably stopped; signalling an error is not retirement. + crate::agx_memory::publish(); + let profile=crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming); + let before=if profile {crate::m3_runtime::physical_counter()} else {0}; + self.completion.signal(); + if profile { + let after=crate::m3_runtime::physical_counter(); + let status=match result {Ok(())=>0,Err(e)=>e.to_errno()}; + pr_info!("M3_PACKET_FENCE vm={} packet={:#x} commands={} signal_before_tick={} signal_after_tick={} status={} counter_hz=24000000\n", + self.vm.id(),self.profile_id(),self.commands.len(),before,after,status); + } + } +} +pub(crate) struct Job {shared:Shared,packet:Arc} +/// Timeout state of a [`Job`], owned by the scheduler's timeout worker. +pub(crate) struct JobTimeout {shared:Shared,packet:Arc} +impl sched::JobImpl for Job { + type TimeoutData = JobTimeout; + fn timeout_data(&self)->JobTimeout { + JobTimeout {shared:self.shared.clone(),packet:self.packet.clone()} + } + fn run(job:&mut Self)->Result> { + // Queue the packet and return its fence; the completion worker publishes and retires + // it, so the scheduler can hand over further ready jobs meanwhile. + if let Err(e)=crate::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) { + pr_err!("M3 scheduler: submission failed {:?}\n",e); + // submit() failed before this packet could be published. + job.packet.release_guard();job.packet.finish(Err(e)); + } + Ok(Some(Fence::from_fence(&job.packet.completion))) + } + fn timed_out(job:&mut JobTimeout,_finished:bool)->sched::Status { + // Retain the publishing branch's explicit diagnostic override. + if !crate::m3_params::timeout_nohang() { + if let Some(r)=Option::as_mut(&mut *job.shared.lock()) {r.health().mark_failed();} + crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + job.packet.finish(Err(ETIMEDOUT));return sched::Status::NoDevice; + } + let guard=job.shared.lock(); + let Some(runtime)=guard.as_ref() else { + job.packet.finish(Err(ENODEV)); + return sched::Status::NoDevice; + }; + // The scheduler's deadline covers the whole packet, including time queued behind + // other batches. The completion worker owns each batch's bounded hardware-completion + // checks. Reinsert a healthy packet instead of turning a late timeout into a global + // device loss or corrupting an already signaled fence. + if runtime.health().healthy() { return sched::Status::NoHang; } + drop(guard); + crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + job.packet.finish(Err(EIO)); + sched::Status::NoDevice + } + fn cancel(job:&mut Self) { + crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + job.packet.finish(Err(ECANCELED)); + } +} +struct AddressSpace<'a>(&'a mmu::Vm); +impl agx_uapi::GpuAddressSpace for AddressSpace<'_> { + fn covers(&self,address:u64,size:u64,access:agx_uapi::GpuAccess)->bool { + let Some(end)=address.checked_add(size) else {return false;}; + let (r,w)=match access {agx_uapi::GpuAccess::Read=>(true,false), + agx_uapi::GpuAccess::Write=>(false,true),agx_uapi::GpuAccess::ReadWrite=>(true,true)}; + address>=0x4000 && end<=(1u64<<42)-0x8000 + && !self.0.driver_range_overlaps(address..end) && self.0.covers_range(address,size,r,w) + } +} +pub(crate) struct Queue { + entity:sched::Entity,_scheduler:Arc>, + shared:Shared,vm:mmu::Vm,usc:u64,fences:FenceContexts, + /// Whether the SoC advertises early tiling (`m3_soc::Features::fragment_dependency`). + early_tiling:bool, + urgency:u8, +} +impl Queue { + pub(crate) fn new(shared:Shared,scheduler:Arc>,vm:mmu::Vm,priority:u32,usc:u64, + early_tiling:bool)->Result { + agx_uapi::QueueUscWindow{base:usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}.validate().map_err(|_|EINVAL)?; + // file.rs passes REALTIME - UAPI priority: 3 is UAPI LOW, 0 is UAPI REALTIME. + let urgency=3u8.saturating_sub(priority.min(3) as u8); + let priority=match priority {3=>sched::Priority::Low,2=>sched::Priority::Normal, + 1=>sched::Priority::High,0=>sched::Priority::Kernel,_=>return Err(EINVAL)}; + Ok(Self {entity:sched::Entity::new(&scheduler,priority)?,_scheduler:scheduler, + shared,vm,usc,fences:FenceContexts::new(1,c_str!("asahi_m3_queue"),kernel::static_lock_class!())?, + early_tiling,urgency}) + } +} +impl queue::Queue for Queue { + fn submit(&mut self,_id:u64,mut syncs:KVec,in_sync_count:usize, + raw:&[u8],objects:Pin<&xarray::XArray>>)->Result { + let vm_job=self.vm.retain_m3_job()?; + let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new(); + while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? { + if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command { + if let Err(e)=validate_render(payload,self.usc,&AddressSpace(&self.vm),self.early_tiling) { + pr_err!("M3 render rejected {:?}: flags={:#x} size={}x{} layers={} samples={} utile={}x{} sample_size={} depth={:#x} stencil={:#x} zls={:#x} samplers={} heap={:#x}\n",e,payload.flags,payload.width,payload.height,payload.layers,payload.samples,payload.utile_width,payload.utile_height,payload.sample_size,payload.depth.base,payload.stencil.base,payload.zls_control,payload.sampler_count,payload.sampler_heap); + return Err(e); + } + agx_uapi::validate_attachments(&AddressSpace(&self.vm),&vertex_attachments).map_err(|_|EINVAL)?; + agx_uapi::validate_attachments(&AddressSpace(&self.vm),&fragment_attachments).map_err(|_|EINVAL)?; + commands.push(Command::Render{command:payload,usc:self.usc},GFP_KERNEL)?; + timestamps.push([Destination::resolve(objects,payload.vertex_timestamps.start)?, + Destination::resolve(objects,payload.vertex_timestamps.end)?, + Destination::resolve(objects,payload.fragment_timestamps.start)?, + Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?; + flushes.push(crate::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; + visibility.push(false,GFP_KERNEL)?; + continue; + } + let agx_uapi::ParsedHardwareCommand::Compute{mut payload,attachments,..}=command else {return Err(ENOTSUPP);}; + // Other engines and helpers need their own M3 firmware translation. + let wide=kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_WIDE_VISIBILITY as u32; + if payload.flags & !wide !=0 || payload.sampler_count!=0 || payload.sampler_heap!=0 { + pr_err!("M3 compute rejected: flags={:#x} samplers={} heap={:#x} attachments={} timestamps={}/{}\n",payload.flags,payload.sampler_count,payload.sampler_heap,attachments.as_slice().len(),payload.timestamps.start.handle,payload.timestamps.end.handle); + return Err(ENOTSUPP); + } + visibility.push(payload.flags & wide !=0,GFP_KERNEL)?; + // The common translator validates the original command/attachment + // addresses; this M3 scheduling hint changes none of them. + payload.flags &= !wide; + flushes.push(encode_attachments(&self.vm,&attachments)?,GFP_KERNEL)?; + let c=agx_uapi::translate_compute_command(payload,attachments, + agx_uapi::QueueUscWindow{base:self.usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}, + &AddressSpace(&self.vm)).map_err(|e| { + pr_err!("M3 compute translation rejected {:?}: stream={:#x}..{:#x} helper={:?}\n",e,payload.control_stream_base,payload.control_stream_end,payload.helper); + EINVAL + })?; + crate::m3_client::check_compute_launches(&self.vm,&AddressSpace(&self.vm),c.control_stream_base,c.control_stream_end)?; + timestamps.push([Destination::resolve(objects,payload.timestamps.start)?, + Destination::resolve(objects,payload.timestamps.end)?,None,None],GFP_KERNEL)?; + commands.push(Command::Compute(crate::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; + } + parser.finish().map_err(|_|EINVAL)?; + let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, + completion:self.fences.new_fence(0,Completion)?.into(), + finish_claimed:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), + vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; + let mut job=self.entity.new_job(1,Job{shared:self.shared.clone(),packet})?; + for sync in syncs.drain(0..in_sync_count) {if let Some(f)=sync.fence {job.add_dependency(f)?;}} + let mut job=job.arm();let finished=job.fences().finished();job.push(); + for mut sync in syncs {if let Some(chain)=sync.chain_fence.take() { + sync.syncobj.add_point(chain,&finished,sync.timeline_value); + } else {sync.syncobj.replace_fence(Some(&finished));}} + Ok(()) + } +} + +fn encode_attachments(vm: &mmu::Vm, list: &agx_uapi::UapiAttachmentList) + -> Result { + use crate::agx_attachments::{Attachments, CAPACITY}; + use agx_uapi::{GpuAddressSpace, GpuAccess}; + if list.as_slice().len()>CAPACITY {return Err(E2BIG);} + let mut ranges = [(0, 0); CAPACITY]; + for (range, entry) in ranges.iter_mut().zip(list.as_slice()) { + *range = Attachments::range(entry.address, entry.size).map_err(|_| EINVAL)?; + if !AddressSpace(vm).covers(range.0, range.1, GpuAccess::Write) { + return Err(EINVAL); + } + } + let mut out=Attachments::new(&ranges[..list.as_slice().len()]).map_err(|_| EINVAL)?; + // G15S RTKit-2419 uses the legacy 0x17 cache attribute; retain the + // M4 range normalization and ownership checks, not its 0x18 attribute. + for i in 0..list.as_slice().len() {out.0[i*16+12..i*16+14].copy_from_slice(&0x17u16.to_le_bytes());} + Ok(out) +} + +fn validate_render(r: agx_uapi::UapiRenderCommand, usc: u64, space: &AddressSpace<'_>, + early_tiling: bool) -> Result { + use agx_uapi::{GpuAddressSpace, GpuAccess}; + if r.sample_size % 8 != 0 {return Err(ENOTSUPP);} + let empty = agx_uapi::UapiHelperProgram { binary:0, config:0, data:0 }; + // Early tiling (render flag bit 5) only where it is advertised. + let stage_scope=if early_tiling && *crate::module_parameters::m3_early_tiling.value()!=0 {1<<5} else {0}; + if r.flags & !(2|16|(1<<18)|stage_scope) != 0 || !matches!(r.samples, 1 | 2 | 4) + || r.sampler_count != 0 || r.sampler_heap != 0 + || r.vertex_helper != empty || r.fragment_helper != empty + || r.stencil.compression_base != 0 || r.stencil.compression_stride != 0 + || r.ppp_control & !0x203 != 0 { return Err(ENOTSUPP); } + // Native 26A428 descriptors witness the ZLS metadata registers and mirrors + // for single-sample, single-layer depth. Keep other layouts unqualified. + if (r.depth.compression_base != 0 || r.stencil.compression_base != 0) + && (r.samples != 1 || r.layers != 1) {return Err(ENOTSUPP);} + // sample_size describes color tilebuffer storage. Depth/stencil-only + // passes legitimately require zero bytes of color storage. + if r.layers == 0 || r.layers > 2048 || r.width == 0 || r.height == 0 || r.width > 16384 || r.height > 16384 + || !matches!(r.utile_width,16|32) || !matches!(r.utile_height,16|32) + || u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)>32768 { + return Err(EINVAL); + } + crate::agx_render_state::compact(r.vdm_base).map_err(|_| EINVAL)?; + if r.vdm_base & 3 != 0 || !space.covers(r.vdm_base,4,GpuAccess::Read) + || r.scissor_base == 0 || r.scissor_base & 7 != 0 || !space.covers(r.scissor_base,8,GpuAccess::Read) { + return Err(EINVAL); + } + for (p,access) in [(r.depth_bias_base,GpuAccess::Read),(r.occlusion_query_base,GpuAccess::Write)] { + if p != 0 && (p & 7 != 0 || !space.covers(p,8,access)) { return Err(EINVAL); } + } + for zls in [r.depth,r.stencil] { agx_uapi::validate_zls(space,zls,r.layers).map_err(|_| EINVAL)?; } + let window = agx_uapi::QueueUscWindow { base:usc,user_start:0x4000,user_end:((1u64 << 42) - 2 * mmu::UAT_PGSZ as u64) }; + for p in [r.background,r.end_of_tile,r.partial_background,r.partial_end_of_tile] { + window.full_program(space,p.usc).map_err(|_| EINVAL)?; + if !space.covers(usc+u64::from(p.usc & !63),64,GpuAccess::Read) { return Err(EINVAL); } + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_thermal.rs b/drivers/gpu/drm/asahi/m3/m3_thermal.rs new file mode 100644 index 0000000000000..59a72187c9699 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_thermal.rs @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Thermal limit of the M3 runtime backend's GPU performance cap (`asahi.m3_thermal`). +//! +//! Without it (`off`, the default with the built-in InitData image) the firmware is given a +//! fixed highest performance state (5 of 8 with device-tree InitData: 1056 of 1380 MHz on +//! J516S) and nothing watches temperatures. The firmware's own die temperature controller is +//! off in the InitData this driver builds. With device-tree InitData the default is `on`; +//! `asahi.m3_thermal=off` restores the fixed cap exactly. +//! +//! With it, the InitData gives the firmware the whole table as its highest state (the +//! "ceiling"), and what bounds the firmware is the runtime cap: the Globals performance-state +//! cap and power-interface state targets, which the firmware keeps reading while it runs (a +//! host moves the cap with a single store, with no message). The runtime cap starts at the +//! fixed cap of `off` (the "safe cap") and never goes below it, so at worst the GPU runs as it +//! does without the limit. +//! +//! - `hold`: the runtime cap stays at the safe cap for the whole boot. The GPU then runs exactly +//! as with `off`, provided the firmware follows the runtime cap; the per-job check below +//! proves or disproves that before `on` is used. +//! - `on`: the runtime cap follows the hottest SoC die temperature, read by a work item every +//! 500 ms while the GPU is busy (every 2 s when idle) from the thermal zones in [`ZONES`] +//! that exist: the SMC's die sensors and the PMP's CPU cluster sensors (see [`ZONES`]): +//! * at or above `critical` (`hot` + 8 C): the safe cap at once; +//! * at or above `hot` (`asahi.m3_thermal_hot`, default 80 C): one state lower, at most every +//! 500 ms and once per reading; +//! * at or below `cool` (`hot` - 10 C): one state higher, at most every 2 s and once per +//! reading, up to the ceiling; +//! * no zone, a zone that cannot be read, a reading outside 10..=125 C, one older than 3 s +//! or one that has not changed at all for 60 s: the safe cap at once. +//! +//! The defaults leave a wide margin below the limits the boot firmware's device tree gives this +//! machine (a 94 C GPU die target and a 111 C SoC-hot temperature), because the sensors may +//! read cooler than the GPU's hottest spot (the PMP's are CPU cluster sensors), and this limit +//! reacts in steps of hundreds of milliseconds while the firmware changes state within the cap +//! much faster. +//! +//! The cap is changed only by the job thread, after a job has retired, right before the state +//! the firmware reports is checked. After a lowering the previous cap is tolerated for 1 s. A +//! reported state above the runtime cap in 20 consecutive checks spanning at least 3 s means +//! the firmware does not follow the runtime cap, so this limit cannot work: the GPU is then +//! marked failed, as for a state above the ceiling. +//! +//! The decisions are in `m3_thermal_policy`, which host tests drive; this module reads the +//! sensors, publishes the cap and logs. + +use core::sync::atomic::{AtomicBool, Ordering}; + +use kernel::{ + bindings, c_str, device, + error::to_result, + new_spinlock, + prelude::*, + str::CStr, + sync::{Arc, SpinLock}, + time::{msecs_to_jiffies, Instant, Monotonic}, + workqueue::{self, impl_has_delayed_work, new_delayed_work, DelayedWork, WorkItem}, +}; + +use crate::{ + m3_adt_config::PstatePolicy, + m3_params::{self, ThermalMode}, + m3_thermal_policy::{Change, Policy, Reading, Source, Why, S}, +}; + +/// The thermal zones read, each the hottest of a set of SoC die sensors: the SMC's die sensors +/// (registered by the SMC hwmon driver) and the PMP's CPU cluster sensors (registered by the +/// PMP report driver once the PMP runs). Zones that do not exist are skipped; a zone that +/// exists but cannot be read fails the reading. The M3 Pro boards (J514S and J516S) need +/// both zones: on a J516S the SMC die sensors alone stayed below 67 C while the PMP hotspot +/// exceeded 85 C and the cap stepped down. Without the PMP running, the cap stays safe. +/// Every T8122 board requires the SMC die zone; T8122 has no PMP hotspot zone. +const ZONES: [&CStr; 2] = [c_str!("macsmc_soc_die"), c_str!("apple_pmp_hotspot")]; +// Do not raise its cap on a partial reconstruction of that sensor set. +const J516_REQUIRED_ZONES: u32 = 3; +fn required_zones() -> u32 { + let Some(root) = kernel::of::root() else { return J516_REQUIRED_ZONES; }; + let Ok(compatible) = root.get_property::>(c_str!("compatible")) else { + return J516_REQUIRED_ZONES; + }; + crate::m3_thermal_policy::required_zones(&compatible) +} + +/// Sensor polling period while the GPU is busy and while it is idle, in ms. +const POLL_BUSY_MS: u32 = 500; +const POLL_IDLE_MS: u32 = 2000; +/// The GPU counts as busy for this long after a job has retired, in ns. +const BUSY_WINDOW_NS: i64 = 5 * S; +/// Lines logging cap changes per boot, and the period of the summary line while busy. +const LOG_LINES: u32 = 256; +const SUMMARY_EVERY_NS: i64 = 30 * S; + +type Time = Instant; + +/// Nanoseconds from `base` to `t`. +fn ns(base: Time, t: Time) -> i64 { + (t - base).as_nanos() +} + +struct SensorState { + reading: Reading, + /// When the GPU last retired a job, in ns from the sensor's base. + busy: i64, +} + +/// Reads [`ZONES`] from a self-rearming delayed work item, so the job thread never waits for the +/// SMC. The work item holds a reference, so it stays valid after the governor is dropped; it +/// then stops at its next run. +#[pin_data] +struct Sensor { + required_zones: u32, + /// The origin of the sensor's and the governor's times. + base: Time, + stopped: AtomicBool, + #[pin] + state: SpinLock, + #[pin] + work: DelayedWork, +} + +impl_has_delayed_work! { + impl HasDelayedWork for Sensor { self.work } +} + +/// One reading of the zone `name`, in millidegrees Celsius; ENODEV if there is no such zone. +fn read_zone(name: &CStr) -> Result { + let mut temp: i32 = 0; + // SAFETY: name is NUL-terminated and temp is a valid output pointer. The + // thermal core acquires a device reference under its registry lock and + // keeps it through sampling, so zone unregistration cannot free it here. + to_result(unsafe { bindings::thermal_zone_get_temp_by_name(name.as_char_ptr(), &mut temp) })?; + Ok(temp) +} + +/// The hottest reading of the zones in [`ZONES`] that exist, and which ones were read. +fn read_zones(required: u32) -> Result<(i32, u32)> { + let mut hottest: Option = None; + let mut zones = 0; + for (i, name) in ZONES.iter().enumerate() { + match read_zone(name) { + Ok(temp) => { + hottest = Some(hottest.map_or(temp, |h| h.max(temp))); + zones |= 1 << i; + } + Err(e) if e == ENODEV => {} + Err(e) => return Err(e), + } + } + if zones & required != required { + return Err(ENODEV); + } + hottest.map(|t| (t, zones)).ok_or(ENODEV) +} + +impl WorkItem for Sensor { + type Pointer = Arc; + + fn run(this: Arc) { + if this.stopped.load(Ordering::Acquire) { + return; + } + let result = read_zones(this.required_zones); + let at = ns(this.base, Time::now()); + let reading = match result { + Ok((mc, zones)) => Reading::Temp { mc, at, zones }, + Err(e) => Reading::Failed { + errno: e.to_errno(), + at, + }, + }; + let busy = { + let mut state = this.state.lock(); + state.reading = reading; + at - state.busy < BUSY_WINDOW_NS + }; + let delay = if busy { POLL_BUSY_MS } else { POLL_IDLE_MS }; + let _ = workqueue::system().enqueue_delayed(this, msecs_to_jiffies(delay)); + } +} + +fn describe(source: Source) -> &'static str { + match source { + Source::Unknown | Source::NoReading => "no reading yet", + Source::Ok(_) => "reading", + Source::Failed(e) if e == ENODEV.to_errno() => "no SoC die temperature zone", + Source::Failed(_) => "a SoC die temperature zone cannot be read", + Source::Stale => "the reading is older than 3 s", + Source::Implausible => "the reading is outside 10..=125 C", + Source::Stuck => "the reading has not changed for 60 s", + } +} + +fn describe_why(why: Why) -> &'static str { + match why { + Why::Critical => "critical", + Why::Hot => "hot", + Why::Cool => "cool", + Why::NoTemperature(source) => describe(source), + } +} + +/// Degrees Celsius with one decimal, for the log. +struct Celsius(i32); + +impl kernel::fmt::Display for Celsius { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let sign = if self.0 < 0 { "-" } else { "" }; + let v = self.0.unsigned_abs(); + write!(f, "{}{}.{} C", sign, v / 1000, (v % 1000) / 100) + } +} + +/// The runtime cap and the thermal limit that moves it. +pub(crate) struct Governor { + mode: ThermalMode, + /// Without the limit (`off`): the cap, which never changes. + fixed: u32, + policy: Policy, + freqs: [u32; 16], + base: Time, + sensor: Option>, + source: Source, + last_summary: Option, + /// The highest state the firmware reported since the last summary. + peak: u32, + lines: u32, +} + +impl Governor { + /// The governor for `policy`. Starts the sensor work item unless the limit is off. + pub(crate) fn new(dev: &device::Device, pstates: &PstatePolicy) -> Result { + let (hot_c, clamped) = m3_params::thermal_hot_param(); + let base = Time::now(); + let mut governor = Governor { + mode: pstates.thermal, + fixed: pstates.max, + policy: Policy::new( + pstates.thermal == ThermalMode::On, + pstates.safe, + pstates.max, + hot_c as i32 * 1000, + ), + freqs: pstates.freqs, + base, + sensor: None, + source: Source::Unknown, + last_summary: None, + peak: 0, + lines: 0, + }; + if governor.mode == ThermalMode::Off { + return Ok(governor); + } + if clamped { + dev_info!( + dev, + "M3: asahi.m3_thermal_hot clamped to {} C ({}..={} C)\n", + hot_c, + m3_params::THERMAL_HOT_MIN, + m3_params::THERMAL_HOT_MAX + ); + } + let sensor = Arc::pin_init( + pin_init!(Sensor { + required_zones: required_zones(), + base, + stopped: AtomicBool::new(false), + state <- new_spinlock!(SensorState { + reading: Reading::None, + busy: 0, + }), + work <- new_delayed_work!("asahi::M3ThermalSensor"), + }), + GFP_KERNEL, + )?; + let _ = workqueue::system().enqueue_delayed(sensor.clone(), msecs_to_jiffies(POLL_BUSY_MS)); + governor.sensor = Some(sensor); + let p = &governor.policy; + let (cool, hot, critical) = p.thresholds(); + match governor.mode { + ThermalMode::Hold => dev_info!( + dev, + "M3: thermal limit holding (asahi.m3_thermal=hold): the firmware may use states 1..={} ({} MHz) and the runtime cap stays at {} ({} MHz) for this boot; the SoC die temperature is logged only\n", + p.ceiling(), + governor.mhz(p.ceiling()), + p.safe(), + governor.mhz(p.safe()) + ), + _ => dev_info!( + dev, + "M3: thermal limit on (asahi.m3_thermal=on): runtime cap {}..={} ({}..={} MHz) from the hottest SoC die temperature of zones macsmc_soc_die and apple_pmp_hotspot: up while at or below {}, down at or above {}, cap {} at or above {} or without a fresh reading\n", + p.safe(), + p.ceiling(), + governor.mhz(p.safe()), + governor.mhz(p.ceiling()), + Celsius(cool), + Celsius(hot), + p.safe(), + Celsius(critical) + ), + } + Ok(governor) + } + + fn mhz(&self, state: u32) -> u32 { + self.freqs.get(state as usize).copied().unwrap_or(0) + } + + fn cap(&self) -> u32 { + match self.mode { + ThermalMode::Off => self.fixed, + ThermalMode::Hold | ThermalMode::On => self.policy.cap(), + } + } + + fn note_source(&mut self, dev: &device::Device, source: Source, temp: Option) { + if source == self.source { + return; + } + self.source = source; + let safe = self.policy.safe(); + match (source, temp) { + (Source::Ok(zones), Some(t)) => { + let smc = if zones & 1 != 0 { + " macsmc_soc_die (SMC die sensors)" + } else { + "" + }; + let pmp = if zones & 2 != 0 { + " apple_pmp_hotspot (PMP CPU cluster sensors)" + } else { + "" + }; + dev_info!( + dev, + "M3: thermal: SoC die temperature {} from zones:{}{}\n", + Celsius(t), + smc, + pmp + ) + } + (Source::Failed(e), _) => dev_info!( + dev, + "M3: thermal: {} (error {}); runtime cap at most {} ({} MHz)\n", + describe(source), + e, + safe, + self.mhz(safe) + ), + _ => dev_info!( + dev, + "M3: thermal: {}; runtime cap at most {} ({} MHz)\n", + describe(source), + safe, + self.mhz(safe) + ), + } + } + + fn summary(&mut self, dev: &device::Device, now: i64, temp: Option) { + if self + .last_summary + .is_some_and(|t| now - t < SUMMARY_EVERY_NS) + { + return; + } + self.last_summary = Some(now); + let peak = core::mem::take(&mut self.peak); + let cap = self.policy.cap(); + let (raised, lowered) = self.policy.counts(); + let (safe, ceiling) = (self.policy.safe(), self.policy.ceiling()); + match temp { + Some(t) => dev_info!( + dev, + "M3: thermal: SoC die {}, runtime cap {} ({} MHz) of {}..={}, firmware peak state {} since the last summary, raised {} and lowered {} times so far\n", + Celsius(t), + cap, + self.mhz(cap), + safe, + ceiling, + peak, + raised, + lowered + ), + None => dev_info!( + dev, + "M3: thermal: no SoC die temperature ({}), runtime cap {} ({} MHz) of {}..={}, firmware peak state {} since the last summary, raised {} and lowered {} times so far\n", + describe(self.source), + cap, + self.mhz(cap), + safe, + ceiling, + peak, + raised, + lowered + ), + } + } + + fn log_change(&mut self, dev: &device::Device, change: Change, temp: Option) { + if self.lines < LOG_LINES { + match temp { + Some(t) => dev_info!( + dev, + "M3: thermal: runtime cap {} -> {} ({} MHz): SoC die {} ({})\n", + change.from, + change.to, + self.mhz(change.to), + Celsius(t), + describe_why(change.why) + ), + None => dev_info!( + dev, + "M3: thermal: runtime cap {} -> {} ({} MHz): {}\n", + change.from, + change.to, + self.mhz(change.to), + describe_why(change.why) + ), + } + } else if self.lines == LOG_LINES { + dev_info!( + dev, + "M3: thermal: further runtime cap changes are not logged\n" + ); + } + self.lines = self.lines.saturating_add(1); + } + + /// Recompute the runtime cap after a job. Returns the new cap if it changed; the caller + /// publishes it. + pub(crate) fn update(&mut self, dev: &device::Device, now: Time) -> Option { + let Some(sensor) = &self.sensor else { + return None; + }; + let reading = sensor.state.lock().reading; + let now = ns(self.base, now); + let (usable, change) = self.policy.update(now, reading); + let temp = usable.ok().map(|(t, _, _)| t); + match usable { + Err(source) => self.note_source(dev, source, None), + Ok((t, _, zones)) => self.note_source(dev, Source::Ok(zones), Some(t)), + } + self.summary(dev, now, temp); + let change = change?; + self.log_change(dev, change, temp); + Some(change.to) + } + + /// The highest state the firmware may report now: the runtime cap, or the highest cap of + /// the grace period after a lowering. + pub(crate) fn allowed(&self, now: Time) -> u32 { + match self.mode { + ThermalMode::Off => self.fixed, + ThermalMode::Hold | ThermalMode::On => self.policy.allowed(ns(self.base, now)), + } + } + + /// Account one check of the reported state; `over` says whether it stayed above + /// [`Self::allowed`], `peak` is the highest state it reported. Fails with ERANGE when the + /// firmware does not follow the runtime cap. + pub(crate) fn note_check( + &mut self, + dev: &device::Device, + now: Time, + over: bool, + peak: u32, + ) -> Result { + self.peak = self.peak.max(peak); + if let Some((count, span)) = self.policy.note_check(ns(self.base, now), over) { + let cap = self.cap(); + dev_err!( + dev, + "M3: thermal: the firmware reported a state above the runtime cap {} ({} MHz) in {} consecutive checks over {} ms: it does not follow the runtime cap; marking the GPU failed\n", + cap, + self.mhz(cap), + count, + span / 1_000_000 + ); + return Err(ERANGE); + } + Ok(()) + } + + /// Note that the GPU retired a job, so the sensor is read at the busy rate. + pub(crate) fn busy(&self, now: Time) { + if let Some(sensor) = &self.sensor { + sensor.state.lock().busy = ns(self.base, now); + } + } +} + +impl Drop for Governor { + fn drop(&mut self) { + if let Some(sensor) = &self.sensor { + sensor.stopped.store(true, Ordering::Release); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs new file mode 100644 index 0000000000000..f9c3ff08006d5 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The parameters of the T8122 start experiment (`t8122_start`): their text, defaults, accepted +//! values and their meaning. Plain data only, so that the host tests run it (tools/asahi). + +/// A parameter that is not given. +pub(crate) const UNSET: u64 = u64::MAX; +/// A parameter whose text is not one of its accepted values. The experiment refuses to start +/// with it rather than fall back to a default. +pub(crate) const INVALID: u64 = u64::MAX - 1; + +/// The Fender window sizes: the ADT sgx reg[1] size less 0x28000 (the difference on T6030), and +/// the whole ADT sgx reg[1]. +pub(crate) const FENDER_RULE: u32 = 0x10_4000; +pub(crate) const FENDER_ADT: u32 = 0x12_c000; +/// The GPU clock-generator offsets in SGX: the T8122 table's (`T8122_IO_MAPPINGS`, the default), +/// and the T6030 one. +pub(crate) const CLOCK_GEN_E1C: u64 = 0xe1_c000; +pub(crate) const CLOCK_GEN_E5C: u64 = 0xe5_c000; +/// The SGX setup write of T6030 and T8122 (offset, value). +pub(crate) const T6030_SGX_SETUP: (usize, u32) = (0xd1_4000, 0x7_0001); +/// The default unit masks: the T8122 table's (`m3_soc::T8122_HWDATA_B`), which are T6030's. +pub(crate) const UNIT_MASK_A: u64 = 0x7_0000_0003; +pub(crate) const UNIT_MASK_B: u32 = 0x7; +/// No unit-mask bit outside these is accepted: the default masks. +pub(crate) const UNIT_MASK_A_LIMIT: u64 = 0x7_0000_0003; +pub(crate) const UNIT_MASK_B_LIMIT: u64 = 0x7; +/// The default performance-state ceiling: the lowest two states. +pub(crate) const PSTATE_CAP: u32 = 2; +/// The highest performance-state ceiling the experiment accepts. The same as the driver's own +/// limit when the fast-die temperature controller is off (`m3_adt_config::ADT_MAX_PSTATE_LIMIT`): +/// J613/J615 have no SoC die temperature zone yet, so there is no thermal feedback above it. +pub(crate) const PSTATE_CAP_MAX: u32 = 5; + +/// Parse a decimal or `0x`-prefixed hexadecimal `u64`. A leading sign is not accepted. +fn number(text: &str) -> Option { + if text.starts_with(['+', '-']) { + return None; + } + match text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")) { + Some(hex) if !hex.starts_with(['+', '-']) => u64::from_str_radix(hex, 16).ok(), + Some(_) => None, + None => text.parse::().ok(), + } +} + +/// Parse a parameter's text: one of `names`, else a number. Anything else, and the two +/// reserved values, give [`INVALID`]. +fn parse(text: &str, names: &[(&str, u64)]) -> u64 { + let text = text.trim(); + if let Some(&(_, value)) = names.iter().find(|(name, _)| *name == text) { + return value; + } + number(text).filter(|v| *v < INVALID).unwrap_or(INVALID) +} + +pub(crate) fn parse_number(text: &str) -> u64 { + parse(text, &[]) +} + +pub(crate) fn parse_fender(text: &str) -> u64 { + parse(text, &[("rule", FENDER_RULE as u64), ("adt", FENDER_ADT as u64)]) +} + +pub(crate) fn parse_clkgen(text: &str) -> u64 { + parse(text, &[("e1c", CLOCK_GEN_E1C), ("e5c", CLOCK_GEN_E5C), ("none", 0)]) +} + +pub(crate) fn parse_sgx_setup(text: &str) -> u64 { + parse(text, &[("none", 0), ("t6030", 1)]) +} + +/// The parameters as given: [`UNSET`], [`INVALID`] or a value. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Raw { + pub(crate) start: u64, + pub(crate) initdata_version: u64, + pub(crate) fender: u64, + pub(crate) clkgen: u64, + pub(crate) sgx_setup: u64, + pub(crate) unit_mask_a: u64, + pub(crate) unit_mask_b: u64, + pub(crate) pstate_cap: u64, +} + +impl Raw { + /// No parameter given. + #[cfg(test)] + pub(crate) const NONE: Raw = Raw { + start: UNSET, + initdata_version: UNSET, + fender: UNSET, + clkgen: UNSET, + sgx_setup: UNSET, + unit_mask_a: UNSET, + unit_mask_b: UNSET, + pstate_cap: UNSET, + }; + + /// Whether any parameter other than `asahi.t8122_start` was given. + pub(crate) fn values_given(&self) -> bool { + [ + self.initdata_version, + self.fender, + self.clkgen, + self.sgx_setup, + self.unit_mask_a, + self.unit_mask_b, + self.pstate_cap, + ] + .iter() + .any(|v| *v != UNSET) + } +} + +/// What `asahi.t8122_start` asks for. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Start { + /// Not given, or 0. + Off, + /// 1. + On, + /// Anything else. + Invalid, +} + +pub(crate) fn start(raw: &Raw) -> Start { + match raw.start { + UNSET | 0 => Start::Off, + 1 => Start::On, + _ => Start::Invalid, + } +} + +/// The GPU clock-generator IO mapping. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum ClockGen { + /// SGX + the offset, mapped read-only for the firmware. + At(u64), + /// No mapping: HwDataB slot 29 stays zero. + Absent, +} + +/// The values of one boot. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Values { + pub(crate) initdata_version: u64, + pub(crate) fender: u32, + pub(crate) clock_gen: ClockGen, + pub(crate) sgx_setup: Option<(usize, u32)>, + pub(crate) unit_mask_a: u64, + pub(crate) unit_mask_b: u32, + pub(crate) pstate_cap: u32, +} + +/// A parameter given with a value it does not accept: its name and what it accepts. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Refusal { + pub(crate) name: &'static str, + pub(crate) accepts: &'static str, +} + +/// A parameter's value, its default when not given; None when invalid. +fn given(param: u64, default: u64) -> Option { + match param { + UNSET => Some(default), + INVALID => None, + v => Some(v), + } +} + +/// Resolve the parameters. The first parameter with a value it does not accept is the refusal. +pub(crate) fn resolve(raw: &Raw) -> Result { + let refuse = |name, accepts| Refusal { name, accepts }; + let initdata_version = given(raw.initdata_version, crate::m3_firmware::G15_V14_8_3_INITDATA) + .filter(|v| *v != 0) + .ok_or(refuse("t8122_initdata_version", "a nonzero number, at most 0xfffffffffffffffd"))?; + let fender = match given(raw.fender, FENDER_RULE as u64) { + Some(v) if v == FENDER_RULE as u64 || v == FENDER_ADT as u64 => v as u32, + _ => return Err(refuse("t8122_fender", "0x104000 or rule, 0x12c000 or adt")), + }; + let clock_gen = match given(raw.clkgen, CLOCK_GEN_E1C) { + Some(0) => ClockGen::Absent, + Some(v) if v == CLOCK_GEN_E1C || v == CLOCK_GEN_E5C => ClockGen::At(v), + _ => return Err(refuse("t8122_clkgen", "e1c, e5c or none")), + }; + let sgx_setup = match given(raw.sgx_setup, 1) { + Some(0) => None, + Some(1) => Some(T6030_SGX_SETUP), + _ => return Err(refuse("t8122_sgx_setup", "none or t6030")), + }; + let unit_mask_a = given(raw.unit_mask_a, UNIT_MASK_A) + .filter(|v| *v != 0 && *v & !UNIT_MASK_A_LIMIT == 0) + .ok_or(refuse("t8122_unit_mask_a", "nonzero, no bits outside 0x700000003"))?; + let unit_mask_b = given(raw.unit_mask_b, UNIT_MASK_B as u64) + .filter(|v| *v != 0 && *v & !UNIT_MASK_B_LIMIT == 0) + .ok_or(refuse("t8122_unit_mask_b", "nonzero, no bits outside 0x7"))? as u32; + let pstate_cap = given(raw.pstate_cap, PSTATE_CAP as u64) + .filter(|v| (1..=u64::from(PSTATE_CAP_MAX)).contains(v)) + .ok_or(refuse("t8122_pstate_cap", "1 to 5"))? as u32; + Ok(Values { + initdata_version, + fender, + clock_gen, + sgx_setup, + unit_mask_a, + unit_mask_b, + pstate_cap, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn with(edit: impl Fn(&mut Raw)) -> Raw { + let mut raw = Raw { start: 1, ..Raw::NONE }; + edit(&mut raw); + raw + } + + #[test] + fn nothing_given_is_off_and_gives_no_values() { + assert_eq!(start(&Raw::NONE), Start::Off); + assert!(!Raw::NONE.values_given()); + assert_eq!(start(&with(|r| r.start = 0)), Start::Off); + assert_eq!(start(&with(|_| {})), Start::On); + assert_eq!(start(&with(|r| r.start = 2)), Start::Invalid); + assert_eq!(start(&with(|r| r.start = INVALID)), Start::Invalid); + assert!(!with(|_| {}).values_given()); + assert!(with(|r| r.pstate_cap = 3).values_given()); + assert!(with(|r| r.fender = INVALID).values_given()); + } + + #[test] + fn defaults_are_the_plan_baseline() { + let v = resolve(&with(|_| {})).unwrap(); + assert_eq!(v.initdata_version, 0x0c08_e21e_8380_0490); + assert_eq!(v.fender, 0x10_4000); + assert_eq!(v.clock_gen, ClockGen::At(0xe1_c000)); + assert_eq!(v.sgx_setup, Some((0xd1_4000, 0x7_0001))); + assert_eq!((v.unit_mask_a, v.unit_mask_b), (0x7_0000_0003, 7)); + assert_eq!(v.pstate_cap, 2); + } + + #[test] + fn every_alternative_resolves() { + let v = resolve(&with(|r| { + r.initdata_version = parse_number("0x123456789abcdef0"); + r.fender = parse_fender("adt"); + r.clkgen = parse_clkgen("e5c"); + r.sgx_setup = parse_sgx_setup("none"); + r.unit_mask_a = parse_number("1"); + r.unit_mask_b = parse_number("0x1"); + r.pstate_cap = parse_number("5"); + })) + .unwrap(); + assert_eq!(v.initdata_version, 0x1234_5678_9abc_def0); + assert_eq!(v.fender, 0x12_c000); + assert_eq!(v.clock_gen, ClockGen::At(0xe5_c000)); + assert_eq!(v.sgx_setup, None); + assert_eq!((v.unit_mask_a, v.unit_mask_b, v.pstate_cap), (1, 1, 5)); + let v = resolve(&with(|r| r.clkgen = parse_clkgen("none"))).unwrap(); + assert_eq!(v.clock_gen, ClockGen::Absent); + // Numbers and names give the same values. + assert_eq!(parse_fender("0x104000"), parse_fender("rule")); + assert_eq!(parse_fender(" 0x12C000 "), parse_fender("adt")); + assert_eq!(parse_clkgen("0xe5c000"), parse_clkgen("e5c")); + assert_eq!(parse_clkgen("0xe1c000"), parse_clkgen("e1c")); + assert_eq!(parse_clkgen("0"), parse_clkgen("none")); + assert_eq!(parse_sgx_setup("1"), parse_sgx_setup("t6030")); + } + + #[test] + fn unaccepted_values_refuse_and_name_the_parameter() { + let cases: [(fn(&mut Raw), &str); 15] = [ + (|r| r.initdata_version = parse_number("g15s"), "t8122_initdata_version"), + (|r| r.initdata_version = 0, "t8122_initdata_version"), + (|r| r.fender = parse_fender("0x144000"), "t8122_fender"), + (|r| r.fender = parse_fender("big"), "t8122_fender"), + (|r| r.clkgen = parse_clkgen("0xe00000"), "t8122_clkgen"), + (|r| r.clkgen = parse_clkgen("e5"), "t8122_clkgen"), + (|r| r.sgx_setup = parse_sgx_setup("2"), "t8122_sgx_setup"), + (|r| r.unit_mask_a = parse_number("0x700000007"), "t8122_unit_mask_a"), + (|r| r.unit_mask_a = 0, "t8122_unit_mask_a"), + (|r| r.unit_mask_b = parse_number("0xf"), "t8122_unit_mask_b"), + (|r| r.unit_mask_b = 0, "t8122_unit_mask_b"), + (|r| r.pstate_cap = 0, "t8122_pstate_cap"), + (|r| r.pstate_cap = 6, "t8122_pstate_cap"), + (|r| r.pstate_cap = 16, "t8122_pstate_cap"), + (|r| r.pstate_cap = parse_number("-1"), "t8122_pstate_cap"), + ]; + for (edit, name) in cases { + assert_eq!(resolve(&with(edit)).unwrap_err().name, name); + } + } + + #[test] + fn pstate_cap_accepts_one_to_five_only() { + for cap in 1..=PSTATE_CAP_MAX { + assert_eq!(resolve(&with(|r| r.pstate_cap = u64::from(cap))).unwrap().pstate_cap, cap); + } + for cap in [0u64, 6, 8, 15, 16] { + assert!(resolve(&with(|r| r.pstate_cap = cap)).is_err(), "cap {cap}"); + } + assert_eq!(PSTATE_CAP_MAX, 5); + } + + #[test] + fn a_leading_sign_is_rejected() { + for text in ["+1", "-1", "0x+1", "0x-1", "+0x10"] { + assert_eq!(parse_number(text), INVALID, "{text:?}"); + } + assert!(resolve(&with(|r| r.pstate_cap = parse_number("+3"))).is_err()); + } + + #[test] + fn reserved_numbers_and_garbage_parse_as_invalid() { + for text in ["", "0x", "x1", "18446744073709551615", "18446744073709551614", "0x1g"] { + assert_eq!(parse_number(text), INVALID, "{text:?}"); + } + assert_eq!(parse_number("18446744073709551613"), INVALID - 1); + assert_eq!(parse_number(" 0X10 "), 16); + } + + #[test] + fn default_masks_are_within_the_limits() { + assert_eq!(UNIT_MASK_A & !UNIT_MASK_A_LIMIT, 0); + assert_eq!(u64::from(UNIT_MASK_B) & !UNIT_MASK_B_LIMIT, 0); + assert_eq!(UNIT_MASK_A_LIMIT, 0x7_0000_0003); + assert_eq!(UNIT_MASK_B_LIMIT, 0x7); + } +} diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs new file mode 100644 index 0000000000000..cf0ee658e8da5 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs @@ -0,0 +1,579 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The T8122 (M3, G15G) start, `asahi.t8122_start=1`. +//! +//! The T8122 table (`m3_soc::T8122`) is complete, but the runtime has not passed on an M3 +//! MacBook Air yet, so the SoC is refused unless `asahi.t8122_start=1` arms the start. On a T8122 +//! whose GPU the boot loader handed over (resource admission passed), an armed start uses the +//! table's values, with a performance-state cap. Each value can be overridden for a lab A/B by +//! rebooting with a parameter: +//! +//! | Parameter | Default | Others | +//! |---|---|---| +//! | `asahi.t8122_initdata_version` | `0x0c08e21e83800490`, the G15 14.8.3 version | any | +//! | `asahi.t8122_fender` | `0x104000` (`rule`) | `0x12c000` (`adt`) | +//! | `asahi.t8122_clkgen` | `e1c`: SGX+0xe1c000, read-only | `e5c`: SGX+0xe5c000, read-only; `none` | +//! | `asahi.t8122_sgx_setup` | `t6030`: SGX+0xd14000 = 0x70001 | `none` | +//! | `asahi.t8122_unit_mask_a` | `0x700000003` | nonzero, within `0x700000003` | +//! | `asahi.t8122_unit_mask_b` | `0x7` | nonzero, within `0x7` | +//! | `asahi.t8122_pstate_cap` | `2` | 1 to 5 | +//! +//! The defaults are the T8122 table's values. A parameter given with a value it does not accept +//! refuses the start. Without `asahi.t8122_start=1` nothing here runs, and T8122 is refused. +//! Nothing here applies to any other SoC. + +use core::mem::offset_of; + +use kernel::{device, prelude::*}; + +use crate::{ + fw::initdata::raw, + initdata::G15RuntimeHwDataB, + m3_adt_config::T8122_IO_MAPPINGS, + m3_init_storage::{self as storage, IoMap}, + m3_soc::{IoMapping, Soc, T8122_HWDATA_B}, + pgtable::{prot, Prot}, + t8122_knobs::{ + self as knobs, ClockGen, Start, Values, CLOCK_GEN_E1C, CLOCK_GEN_E5C, FENDER_ADT, + FENDER_RULE, + }, +}; + +/// The GPU register window (SGX) of T8122. +const SGX: u64 = 0x2_9000_0000; +/// Index of the Fender window and of the GPU clock generator in [`T8122_IO_MAPPINGS`]. +const FENDER: usize = 0; +const CLOCK_GEN: usize = 11; +/// The HwDataB slot of the GPU clock generator. +const CLOCK_GEN_SLOT: usize = 29; + + +/// The firmware IO mappings with the Fender window `fender` and the clock generator at +/// SGX + `clock_gen`. +const fn mappings(fender: u32, clock_gen: u64) -> [IoMapping; 12] { + let mut m = T8122_IO_MAPPINGS; + m[FENDER].2 = fender; + m[FENDER].3 = fender; + m[CLOCK_GEN].1 = SGX + clock_gen; + m +} + +/// The runtime IO maps, laid out once for the larger Fender window so that no slot's firmware +/// VA depends on the parameters. +const LAYOUT: [IoMap; 12] = + storage::pack_iomaps(&mappings(FENDER_ADT, CLOCK_GEN_E1C), storage::IOMAP_BASE); + +const fn iomaps(fender: u32, clock_gen: u64) -> [IoMap; 12] { + let mut m = LAYOUT; + m[FENDER].size = fender as usize; + m[CLOCK_GEN].physical = SGX + clock_gen; + m +} + +/// Every Fender window and clock-generator address, in [`Experiment::variant`] order. +static IO_MAPPINGS: [[IoMapping; 12]; 4] = [ + mappings(FENDER_RULE, CLOCK_GEN_E5C), + mappings(FENDER_RULE, CLOCK_GEN_E1C), + mappings(FENDER_ADT, CLOCK_GEN_E5C), + mappings(FENDER_ADT, CLOCK_GEN_E1C), +]; +static IOMAPS: [[IoMap; 12]; 4] = [ + iomaps(FENDER_RULE, CLOCK_GEN_E5C), + iomaps(FENDER_RULE, CLOCK_GEN_E1C), + iomaps(FENDER_ADT, CLOCK_GEN_E5C), + iomaps(FENDER_ADT, CLOCK_GEN_E1C), +]; + +/// Whether each IO map holds its IO mapping: same slot, the mapping's 16 KiB-aligned physical +/// base, its subpage offset, and room for its total size. +const fn maps_cover(mappings: &[IoMapping; 12], maps: &[IoMap; 12]) -> bool { + let mut i = 0; + while i < 12 { + let (slot, phys, total, _, _) = mappings[i]; + let io = maps[i]; + if io.slot != slot + || io.physical + io.offset as u64 != phys + || io.offset + total as usize > io.size + || io.size % 0x4000 != 0 + { + return false; + } + i += 1; + } + true +} + +const _: () = { + // The default variant's IO mappings are the T8122 table's. + let defaults = mappings(FENDER_RULE, CLOCK_GEN_E1C); + let mut j = 0; + while j < 12 { + let (a, b) = (defaults[j], T8122_IO_MAPPINGS[j]); + assert!(a.0 == b.0 && a.1 == b.1 && a.2 == b.2 && a.3 == b.3 && a.4 == b.4); + j += 1; + } + // The T8122 table: the Fender window at the rule size, the clock generator last, at + // SGX+0xe1c000, 16 KiB, read-only. + assert!(T8122_IO_MAPPINGS[FENDER].0 == 0 && T8122_IO_MAPPINGS[FENDER].1 == SGX + 0xd0_0000); + assert!(T8122_IO_MAPPINGS[FENDER].2 == FENDER_RULE); + assert!(T8122_IO_MAPPINGS[CLOCK_GEN].0 == CLOCK_GEN_SLOT); + assert!(T8122_IO_MAPPINGS[CLOCK_GEN].1 == SGX + CLOCK_GEN_E1C); + assert!(T8122_IO_MAPPINGS[CLOCK_GEN].2 == 0x4000 && !T8122_IO_MAPPINGS[CLOCK_GEN].4); + assert!(CLOCK_GEN + 1 == T8122_IO_MAPPINGS.len()); + // Both windows are whole pages and hold the Fender scratch area (+0x60000..+0x80000). + assert!(FENDER_RULE % 0x4000 == 0 && FENDER_ADT % 0x4000 == 0 && FENDER_RULE >= 0x8_0000); + assert!(FENDER_RULE < FENDER_ADT); + // Both clock-generator blocks are whole pages inside the first 16 MiB of SGX. + assert!(CLOCK_GEN_E5C % 0x4000 == 0 && CLOCK_GEN_E1C % 0x4000 == 0); + assert!(CLOCK_GEN_E5C + 0x4000 <= 0x100_0000 && CLOCK_GEN_E1C + 0x4000 <= 0x100_0000); + let mut i = 0; + while i < 4 { + assert!(maps_cover(&IO_MAPPINGS[i], &IOMAPS[i])); + i += 1; + } + // The default unit masks are the T8122 table's, and the limits allow them. + assert!(T8122_HWDATA_B.unit_mask_a == knobs::UNIT_MASK_A); + assert!(T8122_HWDATA_B.unit_mask_b == knobs::UNIT_MASK_B); + assert!(knobs::UNIT_MASK_A & !knobs::UNIT_MASK_A_LIMIT == 0); + assert!(knobs::UNIT_MASK_B as u64 & !knobs::UNIT_MASK_B_LIMIT == 0); + assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_a) == 0x17c0); + assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_b) == 0x17c8); + // An armed start skips `require_complete`. The only `unported` item of the T8122 table is the + // opt-in itself; if another is added, this fails the build until the start accounts for it. + assert!(crate::m3_soc::T8122.unported.len() == 1); +}; + +/// The values of one armed boot. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Experiment(Values); + +/// Whether `soc` is the T8122 table. +pub(crate) fn is_t8122(soc: &Soc) -> bool { + core::ptr::eq(soc, &crate::m3_soc::T8122) +} + +impl Experiment { + /// Index of this boot's IO-mapping table in [`IO_MAPPINGS`] and [`IOMAPS`]. + fn variant(&self) -> usize { + let adt = usize::from(self.0.fender == FENDER_ADT); + let e1c = usize::from(self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C)); + 2 * adt + e1c + } + + /// How many IO mappings this boot gives the firmware: all, or all but the clock generator. + fn mapping_count(&self) -> usize { + match self.0.clock_gen { + ClockGen::At(_) => CLOCK_GEN + 1, + ClockGen::Absent => CLOCK_GEN, + } + } + + /// The InitData version given to the firmware (`asahi.t8122_initdata_version`). + pub(crate) fn initdata_version(&self) -> u64 { + self.0.initdata_version + } + + /// The firmware IO mappings of the InitData. + pub(crate) fn io_mappings(&self) -> &'static [IoMapping] { + &IO_MAPPINGS[self.variant()][..self.mapping_count()] + } + + /// The runtime's mapping of each of them. With the default Fender window and clock generator + /// they are the T8122 table's own IO maps. + pub(crate) fn iomaps(&self) -> &'static [IoMap] { + if self.0.fender == FENDER_RULE && self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C) { + return &crate::m3_adt_config::T8122_IOMAPS; + } + &IOMAPS[self.variant()][..self.mapping_count()] + } + + /// The register block the IO mapping of `slot` must lie in, given the HwConfig's block + /// `base` for it: the selected clock generator for slot 29. + pub(crate) fn io_block(&self, slot: usize, base: u64) -> u64 { + match self.0.clock_gen { + ClockGen::At(offset) if slot == CLOCK_GEN_SLOT => SGX + offset, + _ => base, + } + } + + /// The HwDataB slots the firmware maps read-only, as a bit mask: the clock generator's. + pub(crate) fn read_only_slots(&self) -> u32 { + match self.0.clock_gen { + ClockGen::At(_) => 1 << CLOCK_GEN_SLOT, + ClockGen::Absent => 0, + } + } + + /// The runtime HwDataB words: the T8122 table's. The unit masks are written over them + /// ([`Self::unit_masks`]). + pub(crate) fn hwdata_b(&self) -> &'static G15RuntimeHwDataB { + &T8122_HWDATA_B + } + + /// The HwDataB unit masks A (+0x17c0) and B (+0x17c8). + pub(crate) fn unit_masks(&self) -> (u64, u32) { + (self.0.unit_mask_a, self.0.unit_mask_b) + } + + /// The SGX write (offset, value) made before the firmware starts, if any. + pub(crate) fn sgx_setup(&self) -> Option<(usize, u32)> { + self.0.sgx_setup + } + + /// The highest performance state the firmware may use (`asahi.t8122_pstate_cap`). + pub(crate) fn pstate_cap(&self) -> u32 { + self.0.pstate_cap + } +} + +/// Firmware page protection of the IO map of `slot`, with the read-only slots `read_only`. +pub(crate) fn mmio_prot(read_only: u32, slot: usize) -> Prot { + if slot < 32 && read_only & (1 << slot) != 0 { + prot::PROT_FW_MMIO_RO + } else { + prot::PROT_FW_MMIO_RW + } +} + +/// Decide, after resource admission, whether `soc` starts with the experiment values. +/// +/// Any SoC but T8122: Ok(None), silently. T8122 without `asahi.t8122_start=1`: Ok(None), so the +/// caller refuses it as before (one extra line when experiment parameters were given). T8122 +/// with `asahi.t8122_start=1`: the experiment, after logging every value, or a refusal for a +/// parameter value it does not accept. +pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> { + if !is_t8122(soc) { + return Ok(None); + } + let raw = crate::m3_params::t8122_params(); + match knobs::start(&raw) { + Start::On => {} + Start::Off => { + if raw.values_given() { + dev_info!( + dev, + "M3 G15G start: asahi.t8122_* values given without asahi.t8122_start=1; ignored\n" + ); + } + return Ok(None); + } + Start::Invalid => { + dev_err!( + dev, + "M3 G15G start: refused: asahi.t8122_start has a value it does not accept (0 or 1); GPU startup disabled\n" + ); + return Err(ENODEV); + } + } + // Admission has required the boot loader's handoff: its firmware segments and reserved + // regions, and on this SoC its power configuration. + if !soc.power_from_boot_loader || soc.firmware.is_none() || soc.hwcfg.is_none() { + dev_err!(dev, "M3 G15G start: refused: the T8122 table is not the one this experiment expects\n"); + return Err(ENODEV); + } + let v = knobs::resolve(&raw).map_err(|refusal| { + dev_err!( + dev, + "M3 G15G start: refused: asahi.{} has a value it does not accept ({}); GPU startup disabled\n", + refusal.name, + refusal.accepts + ); + ENODEV + })?; + let e = Experiment(v); + dev_warn!( + dev, + "M3 G15G start: armed (asahi.t8122_start=1, boot loader handoff admitted): initdata_version={:#018x} fender={:#x} clkgen={} sgx_setup={} unit_mask_a={:#x} unit_mask_b={:#x} pstate_cap={}\n", + v.initdata_version, + v.fender, + clkgen_label(v.clock_gen, v.fender), + match v.sgx_setup { + Some(_) => "t6030(sgx+0xd14000=0x70001)", + None => "none", + }, + v.unit_mask_a, + v.unit_mask_b, + v.pstate_cap + ); + dev_warn!( + dev, + "M3 G15G start: the T8122 table's InitData words, HwData placement and register lists; {} IO mappings at firmware VA {:#x}\n", + e.mapping_count(), + storage::IOMAP_BASE + ); + // J613/J615 have no SoC die temperature zone yet, so a cap above the no-feedback limit runs + // the fanless GPU without any temperature feedback (`m3_thermal` holds the cap but gets no + // reading). The runtime cap and the firmware ceiling stay equal, because the cap is at most + // the no-feedback limit. + if v.pstate_cap > knobs::PSTATE_CAP { + dev_warn!( + dev, + "M3 G15G start: pstate_cap={} is above {}: no SoC die temperature zone on this board, so states up to {} run with no temperature feedback\n", + v.pstate_cap, + knobs::PSTATE_CAP, + v.pstate_cap + ); + } + Ok(Some(e)) +} + +/// The armed line's clock-generator field. It says `ro` for the firmware's own slot-29 mapping, +/// and adds that the Fender window reaches the block read-write when the chosen window covers it +/// (SGX+0xe1c000 lies under the 0x12c000 window but not the 0x104000 one; SGX+0xe5c000 under +/// neither). +fn clkgen_label(clock_gen: ClockGen, fender: u32) -> &'static str { + let fender_end = 0xd0_0000u64 + u64::from(fender); + let reaches = |off: u64| off + 0x4000 <= fender_end; + match clock_gen { + ClockGen::At(CLOCK_GEN_E1C) if reaches(CLOCK_GEN_E1C) => { + "e1c(sgx+0xe1c000,ro; also rw in the fender window)" + } + ClockGen::At(CLOCK_GEN_E1C) => "e1c(sgx+0xe1c000,ro)", + ClockGen::At(_) if reaches(CLOCK_GEN_E5C) => { + "e5c(sgx+0xe5c000,ro; also rw in the fender window)" + } + ClockGen::At(_) => "e5c(sgx+0xe5c000,ro)", + ClockGen::Absent if reaches(CLOCK_GEN_E1C) => { + "none (but the fender window reaches sgx+0xe1c000 rw)" + } + ClockGen::Absent => "none", + } +} + +/// Log, in an armed start, that the probe stopped at `stage` before the firmware started. +pub(crate) fn refused(dev: &device::Device, experiment: Option<&Experiment>, stage: &str, error: Error) { + if experiment.is_some() { + dev_err!( + dev, + "M3 G15G start: refused at {} ({:?}); the firmware was not started, see the lines above\n", + stage, + error + ); + } +} + +/// Log, on a T8122 armed with `asahi.t8122_start=1`, that resource admission refused it. +pub(crate) fn not_admitted(dev: &device::Device, soc: &Soc) { + if is_t8122(soc) && knobs::start(&crate::m3_params::t8122_params()) == Start::On { + dev_err!( + dev, + "M3 G15G start: not armed: resource admission refused the GPU (see the lines above); the boot loader did not hand it over completely\n" + ); + } +} + +// One `M3 G15G verdict:` line per outcome of an armed start, so that a tester can tell them +// apart from the log alone. They run on T8122 only (callers check `is_t8122`); nothing here +// changes what the runtime does. + +/// The GPU coprocessor's start or the driver's InitData upload failed (`m3_runtime::Runtime::new`). +/// `started`: the coprocessor ran and offered its endpoints. +pub(crate) fn prepare_verdict(dev: &device::Device, soc: &Soc, started: bool, error: Error) { + if !is_t8122(soc) { + return; + } + if started { + dev_err!( + dev, + "M3 G15G verdict: driver-refused ({:?}): the GPU firmware is up, but the driver did not publish the InitData (UAT, upload checks or thermal setup, see above)\n", + error + ); + } else { + dev_err!( + dev, + "M3 G15G verdict: firmware-boot-failed ({:?}): the GPU coprocessor did not start, or its RTKit did not offer endpoints 0x20/0x21; no InitData was published\n", + error + ); + } +} + +/// The step the firmware boot sequence reached (`m3_runtime::Runtime::boot_inner`). The verdict +/// uses it to name a failure as a publish, a device-control or an InitData-acknowledgement +/// problem, rather than lumping them all under "initdata-rejected". +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum BootStep { + /// About to send, or sending, the InitData publish message (endpoint 0x20). + Publish, + /// A device-control message (0x13 or 9) before the firmware signalled ready. + DeviceControl, + /// Waiting (up to 2 s) for the firmware to write its ready words. + AwaitReady, + /// The firmware signalled ready; a control or check after that. + PostReady, +} + +/// The firmware's answer to the published InitData (`m3_runtime::Runtime::boot`). `accepted`: +/// the firmware wrote its ready words. `step`: the furthest step the boot reached. +pub(crate) fn boot_verdict( + dev: &device::Device, + version: u64, + step: BootStep, + accepted: bool, + crashed: bool, + result: Result, +) { + let Err(e) = result else { + dev_info!( + dev, + "M3 G15G verdict: firmware-running: the firmware accepted the InitData (version {:#x}) and the device controls; no job has run yet\n", + version + ); + return; + }; + // The firmware signalled ready, then a later control or check failed: the InitData was + // accepted, so this is not a rejection. + if accepted || step == BootStep::PostReady { + if crashed { + dev_err!( + dev, + "M3 G15G verdict: firmware-running-check-failed ({:?}): the firmware accepted the InitData (version {:#x}) then crashed during a post-boot control or check; see the crash lines above\n", + e, + version + ); + } else { + dev_err!( + dev, + "M3 G15G verdict: firmware-running-check-failed ({:?}): the firmware accepted the InitData (version {:#x}), then a post-boot control or check failed (see above)\n", + e, + version + ); + } + return; + } + match step { + BootStep::Publish => dev_err!( + dev, + "M3 G15G verdict: publish-failed ({:?}): the InitData publish message (endpoint 0x20) could not be sent; the firmware was not told about the InitData (version {:#x})\n", + e, + version + ), + BootStep::DeviceControl => dev_err!( + dev, + "M3 G15G verdict: device-control-failed ({:?}): a device-control queue, mailbox send or acknowledgement failed after the InitData (version {:#x}) was published; see the device-control diagnostic above\n", + e, + version + ), + // AwaitReady (and PostReady is handled above). + _ if crashed => dev_err!( + dev, + "M3 G15G verdict: initdata-rejected ({:?}): the firmware crashed after the InitData (version {:#x}) was published; see the crash lines above\n", + e, + version + ), + _ => dev_err!( + dev, + "M3 G15G verdict: initdata-rejected ({:?}): the firmware did not acknowledge the InitData (version {:#x}) within 2 s (its ready words stayed unset); see M3 firmware readiness above\n", + e, + version + ), + } +} + +/// A GPU coprocessor setup step failed before the firmware could be started +/// (`m3_runtime::Runtime::new`): the RTKit state, the mailbox transport or the DRM device. +pub(crate) fn coproc_setup_refused(dev: &device::Device, error: Error) { + dev_err!( + dev, + "M3 G15G start: refused at the GPU coprocessor setup (RTKit/mailbox) ({:?}); the firmware was not started\n", + error + ); +} + +/// The firmware ran a job above the performance-state cap (`m3_runtime`, the after-job pstate +/// check). The `M3: firmware ... stays above the cap ...` line precedes this. +pub(crate) fn cap_violated_verdict(dev: &device::Device, error: Error) { + dev_err!( + dev, + "M3 G15G verdict: cap-violated ({:?}): after a job the firmware reported a performance state above the cap; the GPU was marked failed (see the 'stays above the cap' line above)\n", + error + ); +} + +/// A job-path step failed outside the completion wait (`m3_runtime::Runtime::execute`): a submit, +/// a doorbell, or a post-completion read. No dispatch outcome was determined. +pub(crate) fn job_setup_failed_verdict(dev: &device::Device, error: Error) { + dev_err!( + dev, + "M3 G15G verdict: job-failed-before-wait ({:?}): a submit, doorbell or diagnostic step failed, so no dispatch outcome was determined; see the lines above\n", + error + ); +} + +/// A job that failed (`m3_runtime::Inner::fail`). `primary` is ETIMEDOUT when it did not retire +/// within the runtime's per-batch bound; `start_stamp` is the job's first GPU start (CDM +/// dispatch) timestamp, if readable; `pstate` is the GPU performance-state register, if readable. +pub(crate) fn job_failed_verdict( + dev: &device::Device, + primary: Error, + start_stamp: Option, + pstate: Option, + crashed: bool, +) { + if primary != ETIMEDOUT || crashed { + dev_err!( + dev, + "M3 G15G verdict: job-faulted ({:?}): the firmware crashed, reported an error, or the GPU reported a fault while the job ran; see the lines around this one\n", + primary + ); + return; + } + // A nonzero start timestamp means the firmware dispatched the job on the GPU, so this was not + // a dispatch gap however the power state reads now (the GPU may have idled off since). The + // stamp and pstate print as `Some(hex)` or `None`. + if start_stamp.is_some_and(|start| start != 0) { + dev_err!( + dev, + "M3 G15G verdict: job-ran-completion-missed: the job was dispatched (GPU start timestamp {:x?}) but its completion was not seen within the per-batch bound (pstate register {:x?})\n", + start_stamp, + pstate + ); + return; + } + // The GPU start timestamp is 0 (or unreadable): the job was not dispatched. + match pstate { + Some(p) if p & 0xf == 0 => dev_err!( + dev, + "M3 G15G verdict: job-accepted-never-dispatched: the job was not dispatched (GPU start timestamp {:x?}) and the GPU reads powered down (pstate register {:#x})\n", + start_stamp, + p + ), + Some(p) => dev_err!( + dev, + "M3 G15G verdict: job-timed-out-powered: the job did not retire with the GPU powered (pstate register {:#x}, GPU start timestamp {:x?}); see the engine snapshot below\n", + p, + start_stamp + ), + None => dev_err!( + dev, + "M3 G15G verdict: job-timed-out: the job did not retire (pstate register and GPU start timestamp both unreadable)\n" + ), + } +} + +/// The first job of a kind (0 render, 1 compute) retired (`m3_runtime::Runtime::execute`): +/// `span` is its first start and last end GPU timestamp (24 MHz ticks), if readable. +pub(crate) fn job_completed_verdict(dev: &device::Device, kind: usize, gpu_ns: u64, span: Option<[u64; 2]>) { + let what = if kind == 0 { "render" } else { "compute" }; + match span { + Some([start, end]) if start != 0 && end != 0 => dev_info!( + dev, + "M3 G15G verdict: job-completed: the first {} job finished, GPU timestamps {:#x}..{:#x}, {} ns of GPU time\n", + what, + start, + end, + gpu_ns + ), + Some([start, end]) => dev_err!( + dev, + "M3 G15G verdict: job-retired-without-timestamps: the first {} job retired, but its GPU timestamps are {:#x}..{:#x}\n", + what, + start, + end + ), + None => dev_err!( + dev, + "M3 G15G verdict: job-retired-without-timestamps: the first {} job retired, but its GPU timestamps are unreadable\n", + what + ), + } +} From 7660bed77d0b2c5fbf0422f6d12af56c228a385a Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:22:51 -0500 Subject: [PATCH 02/37] Prepare the original context for carried T8122 commit ad52c9b317f6 Keep the public dependency patch unchanged. The separate 7.3 adaptation restores Aurora integration after the carried series. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/Kconfig | 23 +- drivers/gpu/drm/asahi/debug.rs | 2 +- drivers/gpu/drm/asahi/driver.rs | 252 ++---- drivers/gpu/drm/asahi/file.rs | 273 +----- drivers/gpu/drm/asahi/float.rs | 9 +- drivers/gpu/drm/asahi/g15/mod.rs | 71 +- drivers/gpu/drm/asahi/g15/uat.rs | 8 +- drivers/gpu/drm/asahi/gpu.rs | 119 +-- drivers/gpu/drm/asahi/mmu.rs | 1348 ++++++------------------------ rust/kernel/soc/apple/rtkit.rs | 90 +- 10 files changed, 444 insertions(+), 1751 deletions(-) diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 6b28266b6b4d1..958600a91b06b 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -9,7 +9,7 @@ config RUST_DRM_GPUVM select DRM_GPUVM config DRM_ASAHI - tristate "Aurora (DRM support for Apple AGX GPUs)" + tristate "Asahi (DRM support for Apple AGX GPUs)" depends on RUST depends on DRM=y depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) @@ -21,27 +21,8 @@ config DRM_ASAHI select RUST_DRM_GEM_SHMEM_HELPER select RUST_DRM_GPUVM select RUST_APPLE_RTKIT - select CRYPTO_LIB_SHA256 - help - DRM driver for Apple AGX GPUs, from the M1 family onwards. - - To compile this driver as a module, choose M here: the module will - be called aurora. It keeps "asahi" as a module alias. - -config DRM_ASAHI_G16 - bool "Apple M4 (G16G) support (experimental)" - depends on DRM_ASAHI && ARM64 - depends on APPLE_PMP=y || APPLE_PMP=DRM_ASAHI - depends on APPLE_CLPC=y || APPLE_CLPC=DRM_ASAHI || APPLE_CLPC=n - depends on THERMAL_OF help - Rust backend for the qualified J713/T8132 GPU firmware. - Load aurora explicitly after the native display and GFX TVM supplier - are ready. Firmware-visible storage remains retained until a verified - retirement or reboot. Keep the working kernel available during bring-up. - A rendering VM pins about 0.7 GiB; memory-cgroup limits are advisable. - An aborted receipt disables that command class until reboot. - Suspend, hibernation and kexec are blocked while native graphics owns DMA. + DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) config DRM_ASAHI_DEBUG_ALLOCATOR bool "Use debug allocator" diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs index 479d9763cc7ed..50628ade5ab8e 100644 --- a/drivers/gpu/drm/asahi/debug.rs +++ b/drivers/gpu/drm/asahi/debug.rs @@ -84,7 +84,7 @@ pub(crate) enum DebugFlags { /// Update the cached global debug flags from the module parameter pub(crate) fn update_debug_flags() { - let flags = module_parameters::debug_flags.value(); + let flags = *module_parameters::debug_flags.value(); DEBUG_FLAGS.store(flags, Ordering::Relaxed); } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index d6e461a7c3210..c092595180cf8 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,13 +45,10 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { - binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) g16: Option>, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. raw_dev: *mut bindings::device, @@ -79,18 +76,12 @@ pub(crate) type AsahiDevRef = ARef; pub(crate) type AsahiRegData<'drm> = AsahiDrmRegistrationData<'drm>; /// DRM Driver metadata -/// -/// The major version names the UAPI generation in include/uapi/drm/asahi_drm.h -/// and only changes if that ABI ever breaks. The minor version goes up with -/// each new ioctl, flag or feature bit, so userspace can tell what a kernel -/// accepts without probing. The patch level marks fixes userspace may need to -/// detect. const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { - major: 1, + major: 0, minor: 0, patchlevel: 0, - name: c"aurora", - desc: c"Aurora GPU for Apple AGX", + name: c"asahi", + desc: c"Apple AGX Graphics", }; /// DRM Driver implementation for `AsahiDriver`. @@ -105,6 +96,7 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; @@ -144,82 +136,68 @@ pub(crate) enum ProbeConfig { M3(crate::m3::Chip), } -// OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on -// its entries, so the entries that the native G15 backend owns when it is -// built are passed in by configuration. -macro_rules! asahi_of_device_table { - ($($g15_owned:tt)*) => { - kernel::of_device_table!( - OF_TABLE, - ::IdInfo, - [ - ( - of::DeviceId::new(c"apple,agx-t8140"), - ProbeConfig::G17(&hw::t8140::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6040"), - ProbeConfig::G17(&hw::t6040::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8103"), - ProbeConfig::Legacy(&hw::t8103::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8112"), - ProbeConfig::Legacy(&hw::t8112::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6000"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) - ), - ( - of::DeviceId::new(c"apple,agx-t6001"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) - ), - ( - of::DeviceId::new(c"apple,agx-t6002"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) - ), - ( - of::DeviceId::new(c"apple,agx-t6020"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) - ), - ( - of::DeviceId::new(c"apple,agx-t6021"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) - ), - ( - of::DeviceId::new(c"apple,agx-t6022"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) - ), - ( - of::DeviceId::new(c"apple,agx-t8122"), - ProbeConfig::M3(crate::m3::Chip::T8122) - ), - $($g15_owned)* - ( - of::DeviceId::new(c"apple,agx-t6032"), - ProbeConfig::M3(crate::m3::Chip::T6032) - ), - ] - ); - }; -} - -#[cfg(not(CONFIG_DRM_ASAHI_G15))] -asahi_of_device_table!( - ( - of::DeviceId::new(c"apple,agx-t6030"), - ProbeConfig::M3(crate::m3::Chip::T6030) - ), - ( - of::DeviceId::new(c"apple,agx-t6031"), - ProbeConfig::M3(crate::m3::Chip::T6031) - ), +// OF Device ID table. +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c"apple,agx-t8140"), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8103"), + ProbeConfig::Legacy(&hw::t8103::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8112"), + ProbeConfig::Legacy(&hw::t8112::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6000"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) + ), + ( + of::DeviceId::new(c"apple,agx-t6001"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) + ), + ( + of::DeviceId::new(c"apple,agx-t6002"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) + ), + ( + of::DeviceId::new(c"apple,agx-t6020"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) + ), + ( + of::DeviceId::new(c"apple,agx-t6021"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) + ), + ( + of::DeviceId::new(c"apple,agx-t6022"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) + ), + ( + of::DeviceId::new(c"apple,agx-t8122"), + ProbeConfig::M3(crate::m3::Chip::T8122) + ), + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + ( + of::DeviceId::new(c"apple,agx-t6030"), + ProbeConfig::M3(crate::m3::Chip::T6030) + ), + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + ( + of::DeviceId::new(c"apple,agx-t6031"), + ProbeConfig::M3(crate::m3::Chip::T6031) + ), + ( + of::DeviceId::new(c"apple,agx-t6032"), + ProbeConfig::M3(crate::m3::Chip::T6032) + ), + ] ); -#[cfg(CONFIG_DRM_ASAHI_G15)] -asahi_of_device_table!(); /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { @@ -227,12 +205,14 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + fn unbind<'bound>( + _pdev: &'bound platform::Device>, + this: Pin<&Self::Data<'bound>>, + ) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } - this.close_binding(); } /// Device probe function. @@ -241,7 +221,6 @@ impl platform::Driver for AsahiPlatformDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { debug::update_debug_flags(); - crate::memacct::LEDGER.configure(); dev_info!(pdev.as_ref(), "Probing...\n"); @@ -251,7 +230,6 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; - // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -268,9 +246,6 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -302,20 +277,12 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); - let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; - // The DRM core created this card's debugfs directory when the device was registered; - // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and - // waits for its readers, before the registration data that owns the snapshot is dropped. - if let Some(snapshot) = debug_stats.as_ref() { - crate::stats_raw::register(reg.device(), snapshot); - } - // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); @@ -323,16 +290,13 @@ impl platform::Driver for AsahiPlatformDriver { let raw_stats = crate::sysfs_exports::register( raw_dev, snapshot, - stats.is_some() && module_parameters::stats_export.value() != 0, + stats.is_some() && *module_parameters::stats_export.value() != 0, )?; let driver = AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: None, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, raw_dev, raw_stats, stats, @@ -352,9 +316,6 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -364,12 +325,9 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: None, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -385,9 +343,6 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -403,73 +358,19 @@ impl AsahiPlatformDriver { return Err(error); } }; - let stats = gpu::Gpu::stats_snapshot(&*gpu); + // G17's firmware channel format has no supported stats decoder yet. let raw_dev = pdev.as_ref().as_raw(); - let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); - let raw_stats = match crate::sysfs_exports::register_g17( - raw_dev, - snapshot, - module_parameters::stats_export.value() != 0, - ) { + let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); return Err(error); } }; - if let Err(error) = - crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) - { - dev_warn!( - pdev.as_ref(), - "idle_off_delay_ms not registered: {:?}\n", - error - ); - } Ok(AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, - raw_dev, - raw_stats, - stats, - }) - } -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -impl AsahiPlatformDriver { - pub(crate) fn probe_g16<'bound>( - pdev: &'bound platform::Device>, - cfg: &'static crate::g16::Config, - ) -> Result> { - // SAFETY: the qualified UAT output width constrains this device's DMA. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; - let resources = regs::Resources::new(pdev)?; - let res = regs::G16Resources::new(pdev)?; - let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; - let gpu = crate::g16::Gpu::new(pdev, &drm, cfg, res)?; - let data_gpu = gpu.clone() as Arc; - let data = try_pin_init!(AsahiDrmRegistrationData { - gpu: data_gpu, - resources, - }); - // SAFETY: registration is stored in bound platform data and never forgotten. - let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; - let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = crate::sysfs_exports::register(raw_dev, core::ptr::null(), false)?; - Ok(AsahiPlatformDriverData { - binding, - _device: reg.device().into(), - _reg: reg, - g17: None, - g16: Some(gpu), raw_dev, raw_stats, stats: None, @@ -477,16 +378,9 @@ impl AsahiPlatformDriver { } } -impl AsahiPlatformDriverData<'_> { - pub(crate) fn close_binding(&self) { - self.binding.close(); - } -} - impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { - self.binding.close(); - // `self.raw_dev` was set in `probe` from the platform + // SAFETY: `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 244dca39e41ac..608864949c976 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,8 +16,6 @@ use crate::{ AsahiDrmDevice, // }, gem, - gpu, - memacct, mmu, queue, util::{ @@ -45,7 +43,6 @@ use kernel::sync::{ Mutex, // }; use kernel::time::NSEC_PER_SEC; -use kernel::types::ScopeGuard; use kernel::uaccess::{ UserPtr, UserSlice, // @@ -71,7 +68,6 @@ struct Vm { user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, - gpu: Arc, } impl Drop for Vm { @@ -93,7 +89,6 @@ impl Drop for Vm { pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); } self.vm.bo_deferred_cleanup(); - self.gpu.vm_closed(&self.vm); return; } // When the user Vm is dropped, verify that no user mappings exists. @@ -127,23 +122,10 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, - pub(crate) cumulative: Option, - pub(crate) timeline: bool, } impl SyncItem { - fn parse_one( - file: &DrmFile, - mut data: uapi::drm_asahi_sync, - out: bool, - bundled: bool, - ) -> Result { - if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !out || !bundled || data.timeline_value == 0 { - return Err(EINVAL); - } - data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; - } + fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -164,8 +146,6 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, - cumulative: None, - timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -188,10 +168,12 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: None, + chain_fence: if out { + Some(dma_fence::FenceChain::new()?) + } else { + None + }, timeline_value: data.timeline_value, - cumulative: None, - timeline: true, }) } _ => { @@ -206,52 +188,28 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, - allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; - let mut anchor = None; - // Read each record once so validation and use see the same retained bytes. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); + // SAFETY: We only read this once, so there are no TOCTOU issues. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) })?; // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !allow_cumulative || i < in_count || anchor.is_some() { - return Err(EINVAL); - } - anchor = Some(i as usize); - } - vec.push( - Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, - GFP_KERNEL, - )?; - } - if let Some(anchor) = anchor { - let bundle = crate::g17::sync_output::SignalBundle::new( - &vec[in_count as usize..], - anchor - in_count as usize, - )?; - vec[anchor].cumulative = Some(bundle); - } else { - for sync in &mut vec[in_count as usize..] { - if sync.timeline { - sync.chain_fence = Some(dma_fence::FenceChain::new()?); - } - } + vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; } Ok(vec) @@ -260,7 +218,7 @@ impl SyncItem { #[derive(Clone)] pub(crate) enum Object { - TimestampBuffer(Arc), + TimestampBuffer(Arc), } /// State associated with a client. @@ -275,10 +233,6 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, - /// GPU memory this client created and holds. - mem: Arc, - /// Completed engine time, retained by queues after handle destruction. - usage: Arc, } /// Convenience type alias for our DRM `File` type. @@ -298,35 +252,12 @@ impl drm::file::DriverFile for File { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); let user_range = gpu.user_range(); - let mem = memacct::Client::new()?; - let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) - }) - } - - /// The objects this client created, which `memacct` attributes to it, not every object it - /// has a handle to: an imported or shared buffer is counted once, by its creator. - fn memory_stats(&self) -> Option { - Some(drm::file::MemoryStats { - private: self.mem.committed(), - resident: Some(self.mem.resident()), - ..Default::default() + Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) }) } - fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { - let names = [ - kernel::c_str!("vertex"), - kernel::c_str!("fragment"), - kernel::c_str!("compute"), - ]; - for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { - emit(name, ns); - } - } - fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -336,41 +267,25 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new( - id: u64, - user_range: Range, - mem: Arc, - usage: Arc, - ) -> impl PinInit { - // SAFETY: The initializer supplies aligned, pinned storage for Self. - // XArray::new is infallible; each pinned array is initialized in place, - // then every remaining field is written before returning success. + fn new(id: u64, user_range: Range) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - pin_init::raw_try_init( - raw_vms, - xarray::XArray::>::new(xarray::AllocKind::Alloc1), - )?; + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_vms)?; let raw_queues = addr_of_mut!((*slot).queues); - pin_init::raw_try_init( - raw_queues, - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ), - )?; + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ) + .__pinned_init(raw_queues)?; let raw_objects = addr_of_mut!((*slot).objects); - pin_init::raw_try_init( - raw_objects, - xarray::XArray::>::new(xarray::AllocKind::Alloc1), - )?; + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_objects)?; (*slot).id = id; (*slot).user_range = user_range; - addr_of_mut!((*slot).mem).write(mem); - addr_of_mut!((*slot).usage).write(usage); Ok(()) }) } @@ -422,7 +337,7 @@ impl File { let limits = gpu.queue_limits().ok_or(EINVAL)?; let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; UserSlice::new( - UserPtr::from_addr(data.pointer as usize), + UserPtr::from_addr(data.pointer as _), core::mem::size_of_val(&words), ) .writer() @@ -441,11 +356,8 @@ impl File { return Err(ENODEV); } - let memory_limits = memacct::LEDGER.limits(); let mut params = uapi::drm_asahi_params_global { features: 0, - memory_limit: memory_limits.device, - client_memory_limit: memory_limits.client, vm_start: file.inner().user_range.start, vm_end: file.inner().user_range.end, @@ -465,13 +377,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // Write the retained result to this user slice once. + // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); + UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) + core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) })?; Ok(0) @@ -483,18 +395,15 @@ impl File { if data.pad != 0 || data.size < size as u64 { return Err(EINVAL); } - let user = UserPtr::from_addr(data.pointer as usize); + let user = UserPtr::from_addr(data.pointer as _); let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; if flags != 0 || pad != 0 { return Err(EINVAL); } let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size).writer().write(&[ - vm_id, - 0, - status.reported_error() as u32, - 0, - ])?; + UserSlice::new(user, size) + .writer() + .write(&[vm_id, 0, status.reported_error() as u32, 0])?; Ok(0) } @@ -505,7 +414,7 @@ impl File { return Err(EINVAL); } let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as usize), size) + UserSlice::new(UserPtr::from_addr(data.pointer as _), size) .reader() .read::<[u32; 4]>()?; if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { @@ -569,22 +478,7 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; - // G16 registers a retained backend owner before returning this VM. Any later - // allocation can fail before file::Vm exists to run its normal close path. - let failed_vm = ScopeGuard::new_with_data( - (vm.clone(), user_range.clone(), kernel_range.clone()), - |(vm, user_range, kernel_range)| { - if gpu.tracked_submission() && vm.status().is_some() && !vm.is_closed() { - vm.close_idle_reclaim(); - if let Err(error) = vm.unmap_user_ranges(user_range, kernel_range) { - pr_err!("VM create: closing unpublished VM failed: {:?}\n", error); - } - vm.bo_deferred_cleanup(); - gpu.vm_closed(&vm); - } - }, - ); + let vm = gpu.new_vm(kernel_range.clone())?; let vm_xa = file.inner().vms(); let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -660,12 +554,10 @@ impl File { user_range, kernel_range, _dummy_mapping: dummy_mapping, - gpu: gpu.clone(), }, GFP_KERNEL, )?)?; - drop(failed_vm.dismiss()); data.vm_id = id; Ok(0) @@ -674,7 +566,7 @@ impl File { /// IOCTL: vm_destroy: Destroy a `Vm`. pub(crate) fn vm_destroy( _device: &AsahiDrmDevice, - _reg_data: &driver::AsahiDrmRegistrationData<'_>, + reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_vm_destroy, file: &DrmFile, ) -> Result { @@ -683,6 +575,7 @@ impl File { Err(ENOENT) } else { drop(vm); + reg_data.gpu.vm_closed(); Ok(0) } } @@ -725,48 +618,11 @@ impl File { None }; - let size = gem::checked_object_size(data.size.try_into()?)?; - let commit = memacct::LEDGER - .admit(&file.inner().mem, size as u64) - .inspect_err(|_| { - cls_pr_debug!( - Errors, - "gem_create: {:#x} bytes refused by the GPU memory limits\n", - size - ); - })?; - // Large G17 user objects are populated in full by the first VM_BIND. - // Do not multiply direct reclaim/compaction latency by their folio - // count. Userspace can evict its BO cache and retry after ENOMEM. - let pages_no_reclaim = - gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); - let gem = gem::new_object( - device, - size, - data.flags, - resv_obj, - reclaim_cpu, - pages_no_reclaim, - commit, - )?; + let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; + if reg_data.gpu.prepare_new_objects() { - #[cfg(CONFIG_DRM_ASAHI_G15)] - if reg_data.gpu.as_any().is::() { - gem::prepare_new_object_g15(&gem)?; - } else { - let registered = device.registration_guard().ok_or(ENODEV)?; - let parent: &kernel::platform::Device = registered.as_ref(); - gem::prepare_new_object(&gem, parent.as_ref())?; - } - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - let registered = device.registration_guard().ok_or(ENODEV)?; - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - { - let parent: &kernel::platform::Device = registered.as_ref(); - gem::prepare_new_object(&gem, parent.as_ref())?; - } + gem::prepare_new_object(&gem)?; } - let handle = gem.create_handle(file)?; data.handle = handle; @@ -843,17 +699,11 @@ impl File { } if let Some(limit) = reg_data.gpu.bind_batch_limit() { - if data.num_binds == 0 - || data.num_binds > limit - || data.userptr == 0 - || data.stride % 8 != 0 - || data.stride as usize > kernel::page::PAGE_SIZE - { - cls_pr_debug!(Errors, "vm_bind: Batch outside the GPU's limits\n"); + if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 + || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { return Err(EINVAL); } } - if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -864,7 +714,7 @@ impl File { let size = usize::try_from(data.stride)? .checked_mul(usize::try_from(data.num_binds)?) .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as usize), size).reader(); + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -882,7 +732,7 @@ impl File { let stride = usize::try_from(data.stride)?; let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as usize), size) + UserSlice::new(UserPtr::from_addr(data.userptr as _), size) .reader() .read_all(&mut bytes, GFP_KERNEL)?; let (vm, user_range, kernel_range) = { @@ -1330,33 +1180,11 @@ impl File { return Err(EINVAL); } - // REALTIME stays reserved. HIGH needs CAP_SYS_NICE or DRM master, as amdgpu and - // panthor require for raised queue priority; EACCES lets userspace fall back to MEDIUM. - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { + // TODO: Allow with CAP_SYS_NICE + if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } - // HIGH has only been run on G17; G13 and G14 keep refusing it. - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !reg_data.gpu.as_any().is::() - { - cls_pr_debug!( - Errors, - "queue_create: HIGH priority is not supported on this GPU\n" - ); - return Err(EINVAL); - } - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - // SAFETY: capable() reads the current ioctl task's credentials. - && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } - && !file.is_current_master() - { - cls_pr_debug!( - Errors, - "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" - ); - return Err(EACCES); - } let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -1377,7 +1205,6 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, - file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; @@ -1419,9 +1246,6 @@ impl File { } let gpu = ®_data.gpu; - if data.cmdbuf_size as usize > gpu.max_cmdbuf_size() { - return Err(E2BIG); - } if let Some(manager) = gpu.manager() { manager.update_globals(); } @@ -1454,7 +1278,7 @@ impl File { let mut vec = KVec::new(); let copy_commands = |bytes: &mut KVec| { UserSlice::new( - UserPtr::from_addr(data.cmdbuf as usize), + UserPtr::from_addr(data.cmdbuf as _), data.cmdbuf_size as usize, ) .reader() @@ -1466,13 +1290,8 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = SyncItem::parse_array( - file, - data.syncs, - data.in_sync_count, - data.out_sync_count, - gpu.as_any().is::(), - )?; + let syncs = + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; mod_dev_dbg!( device, @@ -1536,7 +1355,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = u64::from(gpu.base_clock_hz()); + let frequency_hz = gpu.base_clock_hz() as u64; let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 52261d10757d1..46c05699d7722 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -44,11 +44,6 @@ struct F32U { } impl F32 { - /// Return the IEEE-754 binary32 representation. - pub(crate) const fn to_bits(self) -> u32 { - self.0 - } - /// Convert a raw 32-bit representation into an F32 pub(crate) const fn from_bits(u: u32) -> F32 { F32(u) @@ -140,12 +135,12 @@ macro_rules! from_ints { ($u:ty, $i:ty) => { impl From<$i> for F32 { fn from(v: $i) -> F32 { - F32U::from_i64(i64::from(v)).pack() + F32U::from_i64(v as i64).pack() } } impl From<$u> for F32 { fn from(v: $u) -> F32 { - F32U::from_u64(u64::from(v)).pack() + F32U::from_u64(v as u64).pack() } } }; diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 9ffd858598b1c..75d3ecc30ffe5 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for -//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting -//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait +//! for the native display and PMP supplier before booting firmware. T6031 retains explicit +//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -45,7 +45,6 @@ mod adt; mod board; mod boot; -mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -144,23 +143,22 @@ const OF_TABLE: kernel::device_id::IdArray = kernel::device_id::IdArray::new([ ( of::DeviceId::new(c_str!("apple,agx-t6030")), - &&hw::t6030::CONFIG, + &hw::t6030::CONFIG, ), ( of::DeviceId::new(c_str!("apple,agx-t6031")), - &&hw::t6031::CONFIG, + &hw::t6031::CONFIG, ), ]); -// Probe admission requires the native display before firmware starts (T6030 -// also its PMP supplier), so both may be loaded at device creation. +// T6030 probe admission requires the native display and PMP supplier before +// firmware starts. T6031 retains explicit loading until it has an equivalent +// display-order admission gate. kernel::of_device_table!( AUTOLOAD_OF_TABLE, - @none, - [ - of::DeviceId::new(c_str!("apple,agx-t6030")), - of::DeviceId::new(c_str!("apple,agx-t6031")), - ] + MODULE_OF_TABLE_G15, + (), + [(of::DeviceId::new(c_str!("apple,agx-t6030")), ())] ); /// The G15 platform adapter: the shared DRM registration probed through @@ -179,12 +177,6 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); - fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { - if let Some(driver) = &this._driver { - driver.close_binding(); - } - } - /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true @@ -219,6 +211,7 @@ impl platform::Driver for Driver<'_> { }, }) } + } /// Firmware starts only after every fallible module/driver registration has succeeded. @@ -229,8 +222,6 @@ static MODULE_READY: AtomicBool = AtomicBool::new(false); /// No fallible module initialization may follow this call. pub(crate) fn activate() { MODULE_READY.store(true, Ordering::Release); - // SAFETY: The helper only reads the device tree. - unsafe { boot::glue::asahi_g15_report_loader_gpu() }; // SAFETY: Both pinned platform registrations are fully initialized and remain owned // by DriverModule. The C helper borrows their live driver pointer only for this call. unsafe { boot::glue::asahi_g15_reprobe() }; @@ -381,7 +372,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and the client aliases of every user VM. + /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -446,7 +437,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G15: no aurora-g15 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { @@ -459,18 +450,17 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain()?; + gpu.retain(); Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once; `boot::start` already took it before touching the - /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) -> Result { - boot::glue::retain_module() + pub(crate) fn retain(&self) { + boot::glue::retain_module(); } } @@ -545,21 +535,7 @@ impl Firmware { vms, }, ); - let result = firmware.init_graphs().and_then(|_| { - if cfg.platform != hw::Platform::AdtWithPmp { - return Ok(()); - } - // A report before firmware startup is ACKed without enabling bandwidth scaling. - // Reassert it after setup, including the backend's final device control. - // SAFETY: This session retains its acknowledged T6030 AGX power vote. - let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; - if ret == 0 { - Ok(()) - } else { - Err(Error::from_errno(ret)) - } - }); - if let Err(e) = result { + if let Err(e) = firmware.init_graphs() { let Firmware { session, init, @@ -654,7 +630,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_closed(&self, _vm: &mmu::Vm) { + fn vm_closed(&self) { vm::closed(self) } @@ -669,10 +645,9 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, + queue::Queue::new(self, vm, priority, usc_exec_base)?, GFP_KERNEL, )?) } @@ -681,7 +656,7 @@ impl gpu::Gpu for Gpu { &self, bo: gem::ObjectRef, range: Range, - ) -> Result { - crate::timestamp::Buffer::firmware(timestamp::map(self, bo, range)?) + ) -> Result { + timestamp::map(self, bo, range) } } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 53e99f5a772b2..854ff34e729a3 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,10 +25,8 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) -//! and return them after successful retirement. Failed published jobs retain their slots until -//! their last reference drops (until reboot for jobs retained after a fail-stop). -//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` +//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; @@ -98,7 +96,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(None, pa, count, f) } + unsafe { pgtable::with_root_entries(pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index a91f267f9e1e0..90bb25451019e 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,6 +11,7 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. +use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -35,7 +36,6 @@ use kernel::{ }, macros::versions, new_mutex, - new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -45,7 +45,6 @@ use kernel::{ }, Arc, Mutex, - SpinLock, UniqueArc, // }, time::{ @@ -86,7 +85,6 @@ use crate::{ queue, regs, stats, - stats_raw, workqueue, // }; @@ -261,30 +259,18 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, - /// Last raw firmware statistics of this device, exported read-only - /// through `stats_raw`. - raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, + #[allow(clippy::vec_box)] #[pin] - garbage_contexts: Mutex>, + garbage_contexts: Mutex>>>, } /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { - /// Admission bound checked before copying the userspace command stream. - fn max_cmdbuf_size(&self) -> usize { - usize::MAX - } - /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; - /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. - fn raw_stats_snapshot(&self) -> Option>> { - None - } - /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -299,10 +285,6 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; - /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). - fn supports_soft_faults(&self) -> bool { - crate::module_parameters::fault_control.value() == 0xb - } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false @@ -315,14 +297,14 @@ pub(crate) trait Gpu: Send + Sync { fn feature_flags(&self) -> u64 { let mut flags = 0; if self.supports_vm_status() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR); + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; } if self.queue_limits().is_some() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES); + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; } - if self.soft_faults() && self.supports_soft_faults() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); + if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } flags } @@ -330,46 +312,34 @@ pub(crate) trait Gpu: Send + Sync { fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None } - /// The VA range that user VMs may use (`vm_start..vm_end` of GET_PARAMS). + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; + /// Backend-specific user address aperture; other GPUs keep their geometry's range. fn user_range(&self) -> Range { self.uat_geometry().user_usable_range() } - /// Whether this GPU uses the tracked submission model of `file.rs` (besides G17, which - /// `file.rs` recognizes by type): VM_BIND batches are validated as a whole against - /// driver-reserved ranges before any change, and SUBMIT copies the command buffer before - /// resolving syncs because the queue installs a fence in every output sync even on failure. + /// Use tracked command publication and sticky VM status on this backend. fn tracked_submission(&self) -> bool { false } - /// Whether SOFT_FAULTS may be advertised when `fault_control` selects soft faults. + /// Whether faults may be reported as recoverable to userspace. fn soft_faults(&self) -> bool { true } - /// Whether new user objects must be populated, cleaned to the point of coherency and pinned - /// at creation, before any CPU view exists (GPUs whose device is not DMA-coherent and whose - /// default CPU mappings are write-combined). + /// Populate and clean fresh object backing before any WC CPU mapping. fn prepare_new_objects(&self) -> bool { false } - /// The largest VM_BIND batch, if the GPU's UAPI contract bounds it; with a bound, a batch also - /// needs at least one operation, a non-null pointer and a stride that is a multiple of 8 and at - /// most a page (EINVAL otherwise). `None` keeps the shared rules. + /// Optional backend-specific upper bound for a VM_BIND batch. fn bind_batch_limit(&self) -> Option { None } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; + /// A userspace VM handle was closed; release backend-only registry references. + fn vm_closed(&self) {} /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; - /// Create a VM charged to a DRM file when the backend bounds retained bindings. - fn new_vm_for_file(&self, kernel_range: Range, _file_id: u64) -> Result { - self.new_vm(kernel_range) - } - /// Release the backend's handle owner after closing user mapping admission. - /// Accepted jobs and firmware resource leases retain their own references. - fn vm_closed(&self, _vm: &mmu::Vm) {} /// Create a new user command queue. fn new_queue( &self, @@ -378,14 +348,13 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result>; - /// Bind a BO range as a firmware- or CPU-written timestamp buffer. + /// Map a BO as a timestamp buffer fn map_timestamp_buffer( &self, bo: gem::ObjectRef, range: Range, - ) -> Result; + ) -> Result; } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -427,8 +396,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_cfg(&self) -> &'static hw::HwConfig; /// Get the dynamic GPU configuration for this SoC. fn get_dyncfg(&self) -> &hw::DynConfig; - /// Register an unused context, with the objects it retains, for invalidation. - fn free_context(&self, ctx: workqueue::RetiredContext); + /// Register an unused context as garbage + fn free_context(&self, data: KBox>); } /// Private generic trait for functions that don't need to escape this module. @@ -773,7 +742,6 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, - mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) @@ -838,12 +806,6 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); - // Per-device raw statistics snapshot, shared with the debugfs export. - let raw_stats_snap = Arc::pin_init( - new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), - GFP_KERNEL, - )?; - let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -859,7 +821,6 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, - raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -887,7 +848,6 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), - raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1119,7 +1079,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(u64::from(map.writable)), + readwrite: U64(map.writable as u64), }; }); @@ -1169,14 +1129,20 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let guard = self.dev.registration_guard()?; + let Some(guard) = self.dev.registration_guard() else { + return None; + }; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if let Some(fault) = &info { - dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); + if info.is_some() { + dev_err!( + self.dev.as_ref(), + " Fault info: {:#x?}\n", + info.as_ref().unwrap() + ); } info }) @@ -1343,10 +1309,6 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } - fn raw_stats_snapshot(&self) -> Option>> { - Some(self.raw_stats.clone()) - } - fn as_any(&self) -> &dyn Any { self } @@ -1429,7 +1391,6 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1447,7 +1408,6 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, - usage, )?, GFP_KERNEL, )?) @@ -1457,15 +1417,15 @@ impl Gpu for GpuManager::ver { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - crate::timestamp::Buffer::firmware(bo.map_range_into_range( + ) -> Result { + bo.map_range_into_range( self.uat.kernel_vm(), range, IOVA_KERN_TIMESTAMP_RANGE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - )?) + ) } } @@ -1490,7 +1450,7 @@ impl GpuManager for GpuManager::ver { core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); for ctx in garbage_ctx { - if self.invalidate_context(&ctx.data).is_err() { + if self.invalidate_context(&ctx).is_err() { dev_err!( self.dev.as_ref(), "GpuContext: Failed to invalidate GPU context!\n" @@ -1499,9 +1459,6 @@ impl GpuManager for GpuManager::ver { panic!("GPU firmware timed out"); } } - // The firmware no longer knows the context's queues, so their structures may go now, - // ahead of the context data they point to. - core::mem::drop(ctx.retained); } let mut guard = self.alloc.lock(); @@ -1770,7 +1727,7 @@ impl GpuManager for GpuManager::ver { &self.dyncfg } - fn free_context(&self, ctx: workqueue::RetiredContext) { + fn free_context(&self, ctx: KBox>) { let mut garbage = self.garbage_contexts.lock(); if garbage.push(ctx, GFP_KERNEL).is_err() { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f146b2762d812..e868e30baca4c 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -15,7 +15,7 @@ //! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; -use core::mem::{size_of, ManuallyDrop}; +use core::mem::size_of; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ @@ -29,6 +29,7 @@ use core::sync::atomic::{ use kernel::{ addr::{ + DmaAddr, PhysicalAddr, // }, c_str, @@ -108,17 +109,18 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; +use pin_init; + const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; +mod shared; mod lifetime; -mod ppl; mod residency; -mod shared; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; @@ -152,81 +154,23 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// One or two coprocessor instances that never touch the handoff region (G17). + /// Two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at - /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. - /// TTBAT context 1 starts as an alias of the kernel context. + /// primary instance uses the `pagetables` root, the secondary one the root at + /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel + /// context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: Option, + secondary_root_offset: u64, }, } -/// UAT policy of a SoC where it differs from the A18 Pro's. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) struct UatQuirks { - /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and - /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of - /// `struct page`. - /// - /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the - /// regions lose `no-map` and this path goes away. - pub(crate) reserved_tables: bool, - // The following settings keep a SoC on the only policy it was tested with. Each one is - // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. - /// Write back every page-table and context-table change: the GPU table walker is not - /// assumed to snoop the CPU caches. - pub(crate) clean_tables: bool, - /// Invalidate ranges one page at a time, never with range TLBI operations. - pub(crate) page_tlbi: bool, - /// Take the handoff lock around context-table edits. Where the firmware leaves its half of - /// the handoff page alone, its lock bytes are not initialized and are never taken. - pub(crate) handoff_lock: bool, - /// Handoff "current slot" written at initialization. - pub(crate) handoff_slot: u32, - /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in - /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. - pub(crate) native_gpu_permissions: bool, -} - -impl UatQuirks { - /// The A18 Pro and earlier policy. - pub(crate) const DEFAULT: UatQuirks = UatQuirks { - reserved_tables: false, - clean_tables: false, - page_tlbi: false, - handoff_lock: true, - handoff_slot: 0, - native_gpu_permissions: false, - }; -} - const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; -/// M4's nonzero wire incarnation of a host-owned root installation. -const fn binding_wire_generation(incarnation: u64) -> u8 { - if incarnation == 0 { - 0 - } else { - (1 + (incarnation - 1) % 255) as u8 - } -} - -/// Upper word of an application TTBAT slot. M4 exposes firmware upper mappings only in -/// context 0; the existing G15 handoff shares them with user contexts. -const fn application_ttb1(root: u64, asid: usize, map_kernel_to_user: bool) -> u64 { - if map_kernel_to_user { - root | TTBR_VALID | (asid as u64) << TTBR_ASID_SHIFT - } else { - 0 - } -} - /// UAT address space geometry of a given SoC. #[derive(Debug, Copy, Clone)] pub(crate) struct UatGeometry { @@ -366,7 +310,6 @@ struct VmInner { va_range: Range, page_table: UatPageTable, mapping_epoch: Option>, - bind_sequence: Arc, epoch: Option, mm: mm::Allocator<(), KernelMappingInner>, uat_inner: Arc, @@ -412,54 +355,19 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] -pub(crate) struct VmReadGuard(VmExecGuard); - -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] -impl VmReadGuard { - #[cfg(CONFIG_DRM_ASAHI_G15)] - pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { - Vm::covers_range_locked(&mut self.0, address, size, read, write) - } - - pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { - Vm::read_user_locked(&mut self.0, va, buf) - } -} - /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, binding: Option>, bind_token: Option, - /// Nonzero M4 wire incarnation of the root currently owning this slot. - generation: u8, ttb: u64, } /// Data associated with a VM <=> BO pairing #[pin_data] struct VmBoData { - /// Highest sequence assigned to a GPUVA linked to this VM/BO pair. - /// - /// Map steps update this monotonically under the VM reservation before - /// linking the GPUVA. Remap pieces keep their original sequence. - newest_bind: AtomicU64, #[pin] - segments: Mutex>, -} - -/// Copied DMA addresses and the lease covering every alias of this VM/BO association. -/// The lease is dropped only after all aliases have been unmapped and invalidated. -struct SgSegments { - segments: KVVec<(DmaAddress, ResourceSize)>, - _lease: kernel::drm::gem::shmem::SGLease, -} - -impl SgSegments { - fn iter(&self) -> core::slice::Iter<'_, (DmaAddress, ResourceSize)> { - self.segments.iter() - } + segments: Mutex>>, } /// Preallocated resources needed to execute a VM operation. @@ -500,9 +408,6 @@ struct StepContext<'ctx> { /// Preallocated resources that can be used when executing the request. resources: &'ctx mut VmOpResources, - - /// Sequence assigned by the successful map for its related metadata. - bind_sequence: u64, } impl<'ctx> StepContext<'ctx> { @@ -515,7 +420,6 @@ impl<'ctx> StepContext<'ctx> { dev, prot, resources, - bind_sequence: 0, }) } @@ -546,7 +450,6 @@ impl<'ctx> StepContext<'ctx> { #[derive(Default, Clone)] struct VmGpuVa { prot: Option, - bind_sequence: u64, } impl gpuvm::DriverGpuVm for VmInner { @@ -565,11 +468,6 @@ impl gpuvm::DriverGpuVm for VmInner { op: OpMap<'op, Self>, ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { - let bind_sequence = self.bind_sequence.fetch_update( - Ordering::AcqRel, - Ordering::Acquire, - |value| value.checked_add(1), - ).map_err(|_| EOVERFLOW)? + 1; let _mutation = self.mapping_mutation(); let (map_addr, map_length) = (op.addr(), op.length() as usize); let mut iova = op.addr(); @@ -578,16 +476,22 @@ impl gpuvm::DriverGpuVm for VmInner { let one_page = op.flags().contains(gpuvm::GpuVaFlag::REPEAT); - if ctx.dev.is_none() { + let Some(dev) = ctx.dev else { + dev_err!(self.dev.as_ref(), "StepContext missed Bound device!"); return Err(ENODEV); - } - // Bind preparation retained the table's backing before taking the VM reservation. - // Never acquire the object reservation from inside the GPUVM map callback. - let segments = op.vm_bo().data().segments.lock(); - let sgt = segments.as_ref().ok_or(EFAULT)?; - for &(dma_address, dma_len) in sgt.iter() { - let mut addr = dma_address as usize; - let mut len = dma_len as usize; + }; + + let sgt = op.obj().sg_table(dev).inspect_err(|e| { + dev_err!( + self.dev.as_ref(), + "sm_step_map() Failed to get SG table: {:?}", + e + ); + })?; + for sgt_entry in sgt.iter() { + // TODO: proper DMA address/length handling + let mut addr = sgt_entry.dma_address() as usize; + let mut len: usize = sgt_entry.dma_len() as usize; if left == 0 { break; @@ -634,7 +538,6 @@ impl gpuvm::DriverGpuVm for VmInner { iova += len as u64; } - drop(segments); let gpuva = match ctx.preallocated_gpuva() { Ok(gpuva) => gpuva, Err(error) => { @@ -642,18 +545,12 @@ impl gpuvm::DriverGpuVm for VmInner { return Err(error); } }; - op.vm_bo() - .data() - .newest_bind - .fetch_max(bind_sequence, Ordering::Relaxed); let op = op.insert( gpuva, VmGpuVa { prot: Some(ctx.prot), - bind_sequence, }, ); - ctx.bind_sequence = bind_sequence; self.tlbi_contexts_naming_root(map_addr, map_length); Ok(op) @@ -679,7 +576,6 @@ impl gpuvm::DriverGpuVm for VmInner { ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { let orig_prot = op.va_to_unmap().data().prot.ok_or(EINVAL)?; - let orig_bind_sequence = op.va_to_unmap().data().bind_sequence; let orig_invalidated = op.va_to_unmap().flags().contains(GpuVaFlag::INVALIDATED); // Only unmap the hole between prev/next, if they exist @@ -720,8 +616,7 @@ impl gpuvm::DriverGpuVm for VmInner { let _mutation = self.mapping_mutation(); if orig_invalidated { - self.page_table - .discard_partial_map(unmap_start..unmap_end)?; + self.page_table.discard_partial_map(unmap_start..unmap_end)?; } else { self.page_table.unmap_pages(unmap_start..unmap_end)?; } @@ -745,11 +640,9 @@ impl gpuvm::DriverGpuVm for VmInner { [prev_gpuva, next_gpuva], VmGpuVa { prot: Some(orig_prot), - bind_sequence: orig_bind_sequence, }, VmGpuVa { prot: Some(orig_prot), - bind_sequence: orig_bind_sequence, }, ); @@ -764,12 +657,7 @@ impl gpuvm::DriverGpuVm for VmInner { impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!( - self.dev, - "MMU: unmap: {:#x}:{:#x}\n", - va.addr(), - va.length() - ); + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); let _mutation = self.mapping_mutation(); if va.flags().contains(GpuVaFlag::INVALIDATED) { @@ -804,7 +692,14 @@ impl VmInner { // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); - self.finish_failed_map(addr, size as usize); + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + mem::sync(); + self.tlbi_range(asid as u8, addr, size as usize); + mem::sync(); + } + } + self.tlbi_contexts_naming_root(addr, size as usize); if let Err(error) = result { dev_err!( self.dev.as_ref(), @@ -814,19 +709,6 @@ impl VmInner { } } - /// Complete discarded stores and invalidate both the legacy bound slot and - /// every M4/G17 context naming this root. Call only after releasing PPL. - fn finish_failed_map(&self, addr: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - mem::sync(); - self.tlbi_range(asid as u8, addr, size); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size); - } - fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { @@ -881,19 +763,15 @@ impl VmInner { /// /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Adopted M4 roots also remain in the TTBAT after a bind - /// lease is dropped, until the slot is reassigned or the root is destroyed. Include those - /// contexts before allowing an unmapped object's backing to be released. + /// change must reach all of them. Other firmware only uses the bind slot, which the callers + /// already invalidate. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff && !self.uat_inner.m4_adopted { + if self.uat_inner.firmware == UatFirmware::Handoff { mem::sync(); return; } - let contexts = self - .uat_inner - .lock() - .contexts_naming_root(self.ttb(), self.uat_inner.m4_adopted); + let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); self.tlbi_context_mask(iova, size, contexts); } @@ -921,35 +799,10 @@ impl VmInner { .checked_add(node.mapped_size as u64) .ok_or(EOVERFLOW)?; let _mutation = self.mapping_mutation(); - // Allocate detached RAM tables before excluding the firmware. The shadow L2 is not - // the reserved shared-L2; only final leaf stores below already linked L3s are live. self.page_table.prepare_map(node.start()..end)?; - let uat_inner = self.uat_inner.clone(); - let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; let result = self.map_node_prepared(node, prot); - let discard = if result.is_err() { - // Rollback still changes live leaves and must remain under the PPL lease. - self.page_table.discard_partial_map(node.start()..end) - } else { - Ok(()) - }; - // The M4 handoff guard owns UatShared's mutex. Invalidation follows its release. - drop(handoff); if result.is_err() { - self.finish_failed_map(node.start(), node.mapped_size); - } else { - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); - } - if let Err(error) = discard { - if self.uat_inner.m4_adopted { - // The caller retains its node and backing when this latch is set. - self.uat_inner.failed.store(true, Ordering::Release); - } - dev_err!( - self.dev.as_ref(), - "Could not discard failed kernel mapping: {:?}\n", - error - ); + self.rollback_failed_map(node.start(), node.mapped_size as u64); } result } @@ -957,14 +810,7 @@ impl VmInner { /// Install only into the empty range reserved by `map_node`. fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node - .bo - .as_ref() - .ok_or(EINVAL)? - .get()? - .data() - .segments - .lock(); + let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -1022,7 +868,8 @@ impl VmInner { } } - // The caller invalidates negative translations after releasing its PPL lease. + // A context naming this root may be live and hold a negative translation for the range. + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); Ok(()) } } @@ -1030,11 +877,9 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { - device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, - bind_sequence: Arc, lifetime: Option>, residency: Option>, context_bindings: Option>, @@ -1101,7 +946,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff_lock(); + inner.handoff().lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -1109,9 +954,8 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); - inner.publish_slot(id); } - inner.handoff_unlock(); + inner.handoff().unlock(); owned || (low == 0 && high == 0) }; if release { @@ -1152,12 +996,6 @@ impl VmBind { pub(crate) fn slot(&self) -> u32 { self.1 } - - /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn generation(&self) -> u8 { - self.0.binding.lock().generation - } } impl Drop for VmBind { @@ -1198,7 +1036,10 @@ struct MappingBo { owner: ARef>, } impl MappingBo { - fn new(bo: ARef>, owner: ARef>) -> Self { + fn new( + bo: ARef>, + owner: ARef>, + ) -> Self { Self { bo: Some(bo), owner, @@ -1241,90 +1082,29 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's -/// reservation lock, so it must not be dropped where fence producers run. -pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); - -impl TimestampVmap { - pub(crate) fn new(gem: &gem::Object) -> Result { - Ok(Self(gem.owned_vmap::<0>()?)) - } - - fn size(&self) -> usize { - use kernel::drm::gem::BaseObject; - self.0.owner().size() - } - - /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete - /// atomic store; byte stores support unaligned destinations without creating mutable - /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused - /// when the destination is prepared, and ignored here. - pub(crate) fn write(&self, offset: usize, value: u64) { - if offset - .checked_add(size_of::()) - .is_none_or(|end| end > self.size()) - { - return; - } - // Completion runs in task context. Bindings of the same BO share this lock, - // so aligned u64 and overlapping unaligned byte stores never race in Rust. - let _guard = self.0.owner().timestamp_lock.lock(); - let base = self.0.addr.cast::(); - // SAFETY: The complete extent was checked above, inside this pinned vmap. - let pointer = unsafe { base.add(offset) }; - if pointer as usize % core::mem::align_of::() == 0 { - // SAFETY: All eight bytes were checked above; the vmap pins the backing. - // A naturally aligned query result becomes available in one complete store. - let destination = unsafe { &*pointer.cast::() }; - destination.store(value.to_le(), Ordering::Relaxed); - } else { - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: The complete eight-byte extent was checked above. The owned - // vmap pins its backing; byte atomics permit unaligned and overlapping - // timestamp destinations without mutable slice aliases. - let destination = unsafe { &*base.add(offset + index).cast::() }; - destination.store(byte, Ordering::Relaxed); - } - } - // A CPU-retired query can be consumed by a later GPU command. WRITEBACK - // objects use a cacheable vmap, while GPU mappings need not snoop that cache. - // Publish the bytes before signalling the completion fence or queue frontier. - if self.0.owner().is_writeback() { - crate::pgtable::clean_poc(pointer, size_of::()); - } - crate::pgtable::poc_barrier(); - } -} - -/// Prepared CPU timestamp access with its own vmap; destruction happens after completion -/// signalling. -#[cfg(CONFIG_DRM_ASAHI_G16)] +/// Prepared CPU timestamp access; destruction happens after completion signalling. pub(crate) struct TimestampMapping { - mapping: TimestampVmap, + mapping: kernel::drm::gem::shmem::VMapOwned, offset: usize, } -#[cfg(CONFIG_DRM_ASAHI_G16)] impl TimestampMapping { - /// Prepare one bounded CPU timestamp destination on retained GEM backing. - pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { - use kernel::drm::gem::BaseObject; - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { - return Err(ERANGE); - } - Ok(Self { - mapping: TimestampVmap::new(gem)?, - offset, - }) - } - pub(crate) fn write(&self, value: u64) { - self.mapping.write(self.offset, value); + let base = self.mapping.addr.cast::(); + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: timestamp_mapping checked the complete eight-byte extent. + // The owned vmap pins its backing; byte atomics permit unaligned and + // overlapping timestamp destinations without mutable slice aliases. + let destination = unsafe { + &*base.add(self.offset + index).cast::() + }; + destination.store(byte, Ordering::Relaxed); + } } } /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(ManuallyDrop>); +pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); impl KernelMapping { /// Returns the IOVA base of this mapping @@ -1348,22 +1128,14 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - self.0.owner.lock_inner().page_table.translate_iova(address) + lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) } - /// Writes a userspace timestamp through the retained GEM mapping. Aligned results - /// use a complete atomic store; byte stores support unaligned destinations without - /// creating mutable aliases. An unaligned complete timestamp is not atomic. + /// Writes a userspace timestamp through the retained GEM mapping. Byte + /// stores support unaligned destinations and concurrent completion writes + /// without creating mutable aliases. The complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. - #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { - let gem = self.0._gem.as_ref().ok_or(EINVAL)?; - TimestampMapping::new(gem, self.timestamp_offset(offset)?) - } - - /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete - /// extent lies inside both the mapping and the object. - pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1374,12 +1146,10 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - Ok(start) - } - - /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. - pub(crate) fn timestamp_vmap(&self) -> Result { - TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) + Ok(TimestampMapping { + mapping: gem.owned_vmap::<0>()?, + offset: start, + }) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1491,13 +1261,13 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff_lock(); + uat_inner.handoff().lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff_unlock(); + uat_inner.handoff().unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1587,19 +1357,6 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { - // An ASC-cached M4 mapping has no qualified cache-flush teardown. Even an unexpected - // failed construction must retain its complete node/backing instead of invoking the - // unrelated G13/G14 FwCtl protocol. - if self.0.uat_inner.m4_adopted && self.0.prot.is_cached_noncoherent() { - self.0.uat_inner.failed.store(true, Ordering::Release); - pr_err!( - "MMU: retaining cached M4 mapping {:#x}+{:#x} until reboot\n", - self.iova(), - self.size() - ); - return; - } - // This is the main unmap function for UAT mappings. // The sequence of operations here is finicky, due to the interaction // between cached GFX ASC mappings and the page tables. These mappings @@ -1642,27 +1399,8 @@ impl Drop for KernelMapping { self.size() ); - let upper_handoff = match self.0.uat_inner.lock_m4_upper(owner.is_kernel) { - Ok(guard) => guard, - Err(error) => { - dev_err!( - owner.dev.as_ref(), - "MMU: retaining M4 mapping after PPL failure: {:?}\n", - error - ); - return; - } - }; let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { - if self.0.uat_inner.m4_adopted { - self.0.uat_inner.failed.store(true, Ordering::Release); - dev_err!( - owner.dev.as_ref(), - "MMU: retaining M4 mapping after partial unmap failure\n" - ); - return; - } dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -1671,8 +1409,6 @@ impl Drop for KernelMapping { ); } - drop(upper_handoff); - if let Some(asid) = owner.slot() { mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); @@ -1689,11 +1425,6 @@ impl Drop for KernelMapping { // The range returns to the VA allocator once this mapping is gone, so no context that // names this root may keep a stale translation for it. owner.tlbi_contexts_naming_root(self.iova(), self.size()); - drop(_mutation); - drop(owner); - // SAFETY: Every normal path reaches this exactly once, after all GPUVM locks have - // been released. Uncertain M4 paths return earlier, retaining the node and its pins. - unsafe { ManuallyDrop::drop(&mut self.0) }; } } @@ -1702,73 +1433,32 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], - /// Firmware-observed incarnations of application roots installed by the VM slot allocator. - binding_generations: [u64; UAT_NUM_CTX], - binding_published: [bool; UAT_NUM_CTX], - /// Expected lower words of user roots owned by the VM slot allocator. Replaced only by - /// successful host publications, never learned by accepting observed hardware words. - owned_user_roots: [u64; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, - quirks: UatQuirks, } impl UatShared { - /// Takes the handoff lock, where this SoC uses it. - fn handoff_lock(&self) { - if self.quirks.handoff_lock { - self.handoff().lock(); - } - } - - /// Releases the handoff lock taken by [`Self::handoff_lock`]. - fn handoff_unlock(&self) { - if self.quirks.handoff_lock { - self.handoff().unlock(); - } - } - - /// Writes back context-table slot `idx` after it changed, where this SoC needs it. - fn publish_slot(&self, idx: usize) { - if self.quirks.clean_tables { - let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); - pgtable::clean_poc(slot, size_of::()); - pgtable::poc_barrier(); - } - } - /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() + unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { - self.ttbs_rgn - .map - .ptr() - .cast::<[SlotTTBS; UAT_NUM_CTX]>() - .as_ref() - } - .unwrap() + unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64, adopted: bool) -> u64 { + fn contexts_naming_root(&self, ttb: u64) -> u64 { self.ttbs() .iter() .enumerate() - .filter(|(ctx, slot)| { - let expected = tagged_root(ttb, *ctx); - let _ = adopted; // Context 0 is naturally untagged; user contexts carry their ASID. - slot.ttb0.load(Ordering::Acquire) == expected - }) + .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } } @@ -1785,138 +1475,20 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, - /// The stopped J713 handoff: bounded full-barrier PPL and upper words zero in users. - m4_adopted: bool, - /// An uncertain mapping/root is retained; future M4 allocation and publication fail. - failed: AtomicBool, - /// Clean page tables to the point of coherency for a walker that does not snoop the CPU - /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. + #[cfg(CONFIG_DRM_ASAHI_G15)] clean_poc: bool, - /// This SoC's UAT policy. - quirks: UatQuirks, - /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). - reserved: Option>, #[pin] shared: Mutex, #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } -/// A transport owns the sole host-side PPL mutex before touching AP interest. -struct M4Transport<'a> { - shared: Guard<'a, UatShared, MutexBackend>, - start: Instant, - failed: &'a AtomicBool, -} - -impl ppl::Memory for M4Transport<'_> { - fn firmware_interested(&self) -> bool { - self.shared.handoff().lock_fw.load(Ordering::Relaxed) != 0 - } - fn firmware_turn(&self) -> bool { - self.shared.handoff().turn.load(Ordering::Relaxed) != 0 - } - fn ap_interest(&self, interested: bool) { - self.shared - .handoff() - .lock_ap - .store(u8::from(interested), Ordering::Relaxed); - } - fn give_firmware_turn(&self) { - self.shared.handoff().turn.store(1, Ordering::Relaxed); - } - fn barrier(&self) { - mem::sync(); - } - fn expired(&self) -> bool { - self.start.elapsed() > Delta::from_millis(100) - } - fn relax(&self) { - core::hint::spin_loop(); - } - fn fail(&self) { - self.failed.store(true, Ordering::Release); - } -} - -/// Every AP firmware transaction owns the same mutex, including G16 Window operations. -/// M4 drops its protocol guard before its transport releases the host mutex. The legacy -/// handoff path keeps the preexisting protocol and lock ordering for other generations. -#[must_use = "retain the guard until protected table mutations have completed"] -enum HandoffGuard<'a> { - M4(ppl::Guard>), - Legacy(Guard<'a, UatShared, MutexBackend>), -} - -impl HandoffGuard<'_> { - fn shared_mut(&mut self) -> &mut UatShared { - match self { - Self::M4(guard) => &mut guard.memory_mut().shared, - Self::Legacy(shared) => shared, - } - } -} - -impl Drop for HandoffGuard<'_> { - fn drop(&mut self) { - if let Self::Legacy(shared) = self { - shared.handoff_unlock(); - } - } -} - -/// Opaque host-and-firmware lease for G16 window transactions. No caller may -/// independently manipulate the AP-interest or turn words. -#[cfg(CONFIG_DRM_ASAHI_G16)] -#[must_use = "retain the guard throughout the reserved-table window transaction"] -pub(crate) struct M4HandoffGuard<'a> { - _transaction: HandoffGuard<'a>, -} - impl UatInner { /// Take the lock on the shared data and return the guard. fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } - fn check_m4(&self) -> Result { - if self.m4_adopted && self.failed.load(Ordering::Acquire) { - Err(EIO) - } else { - Ok(()) - } - } - - /// Serialize AP callers before taking the firmware exclusion. Timeouts withdraw - /// AP interest while still holding the mutex and latch M4 failure before releasing it. - fn lock_handoff(&self) -> Result> { - self.check_m4()?; - let shared = self.lock(); - self.check_m4()?; - if !self.m4_adopted { - shared.handoff_lock(); - return Ok(HandoffGuard::Legacy(shared)); - } - let transport = M4Transport { - shared, - start: Instant::::now(), - failed: &self.failed, - }; - match ppl::Guard::acquire(transport) { - Ok(guard) => Ok(HandoffGuard::M4(guard)), - Err(()) => Err(ETIMEDOUT), - } - } - - /// No leaf is changed on a failed acquisition. Uncertain mappings retain their backing. - fn lock_m4_upper(&self, upper: bool) -> Result>> { - if upper && self.m4_adopted { - self.lock_handoff().map(Some) - } else { - Ok(None) - } - } - /// Take a lock on a handoff flush slot and return the guard. fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() @@ -1944,9 +1516,7 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 { - core::hint::spin_loop() - } + while self.turn.load(Ordering::Relaxed) != 0 {} self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -1971,9 +1541,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware, slot: u32) -> Result { + fn init(&self, firmware: UatFirmware) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(slot, Ordering::Relaxed); + self.cur_slot.store(0, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -2053,39 +1623,19 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { +fn prefetch_sgt_segments( + dev: &device::Device, + gem: &gem::Object, +) -> Result> { let sgt = crate::gem::held_sg_table(gem, dev)?; + // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(SgSegments { - segments, - _lease: sgt.into_lease(), - }) -} - -/// Reuse cached segments without allocating another copy for each bind. The bind lease was -/// acquired before BO locking: its getter synchronized with any earlier reclaim under resv, -/// and its extra count excludes new eviction until preparation/commit finishes. -fn prepare_sgt_segments( - dev: &device::Device, - gem: &gem::Object, - bo: &gpuvm::GpuVmBo, - _bind_lease: &kernel::drm::gem::shmem::SGLease, -) -> Result { - let needs_segments = bo.data().segments.lock().is_none(); - if needs_segments { - // Never enter the object reservation while holding the BO mutex. - let prefetched = prefetch_sgt_segments(dev, gem)?; - let mut segments = bo.data().segments.lock(); - if segments.is_none() { - segments.replace(prefetched); - } - } - Ok(()) + Ok(segments) } impl Vm { @@ -2108,12 +1658,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, - uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? - .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; + + #[cfg(CONFIG_DRM_ASAHI_G15)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -2139,7 +1689,6 @@ impl Vm { VmBinding { binding: None, bind_token: None, - generation: 0, active_users: 0, ttb: page_table.ttb(), }, @@ -2153,7 +1702,6 @@ impl Vm { } else { None }; - let bind_sequence = Arc::new(AtomicU64::new(0), GFP_KERNEL)?; let unique_gpuvm = gpuvm::GpuVm::new::( c"Asahi::GpuVm", &dev, @@ -2166,7 +1714,6 @@ impl Vm { va_range: va_range.clone(), page_table, mapping_epoch: mapping_epoch.clone(), - bind_sequence: bind_sequence.clone(), epoch: Some(0), mm, uat_inner, @@ -2176,14 +1723,11 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); - let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { - device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, mapping_epoch, - bind_sequence, lifetime: None, residency: None, context_bindings: None, @@ -2196,10 +1740,7 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init( - crate::g17::status::VmStatus::new(), - GFP_KERNEL, - )?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); @@ -2210,18 +1751,15 @@ impl Vm { /// Attach a status and job lifetime tracking before sharing a newly created user VM with /// queues, without the G17 canonical context views, shared-object footprints and idle /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn with_tracked_status(mut self) -> Result { - self.status = Some(Arc::pin_init( - crate::g17::status::VmStatus::new(), - GFP_KERNEL, - )?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); Ok(self) } - /// The identifier this VM was created with (G15 keeps per-VM state keyed by it; G17 timeout - /// attribution compares and logs VMs by it). + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn id(&self) -> u64 { self.id } @@ -2233,18 +1771,9 @@ impl Vm { /// and invalidated to the point of coherency first (never only invalidated: CPU-written lines /// may be dirty), so the copy sees what the GPU or a write-combined CPU view stored. The /// mappings cannot change during the copy (GPUVM exec lock). - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - self.read_lock()?.read_user(va, buf) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - pub(crate) fn read_lock(&self) -> Result { - Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { + let mut inner = lock_vm(&self.gpuvm, None, false)?; let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -2314,7 +1843,9 @@ impl Vm { } pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return false; + }; Self::covers_range_locked(&mut inner, address, size, read, write) } @@ -2329,22 +1860,26 @@ impl Vm { if stride <= block_size { return false; } - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return false; + }; addresses.iter().all(|&address| { let Some(guard) = address.checked_add(block_size) else { return false; }; Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked(&mut inner, guard, stride - block_size, false, false) + && !Self::covers_range_locked( + &mut inner, guard, stride - block_size, false, false, + ) }) } - /// Checks all exact request tuples under one hold of the VM reservation. Access bit0 + /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 /// requests reading and bit1 writing. The epoch names this same snapshot. pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { - // Every mapping change holds the VM reservation, so readers need only that one. The full - // exec lock would also lock every external BO, making each submit O(external BOs). - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return (false, None); + }; let epoch = inner.epoch; let covered = ranges.iter().all(|&(address, size, access)| { Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) @@ -2357,10 +1892,6 @@ impl Vm { self.mapping_epoch.as_ref()?.snapshot() } - fn current_bind_sequence(&self) -> u64 { - self.bind_sequence.load(Ordering::Acquire) - } - /// Get the translation table base for this Vm fn ttb(&self) -> u64 { self.binding.lock().ttb @@ -2396,49 +1927,6 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, - ) -> Result { - self.map_in_range_with_mode( - gem, - object_range, - alignment, - range, - prot, - guard_size, - mm::InsertMode::Best, - ) - } - - /// Lowest-address placement for a monotonic search across several VMs. - pub(crate) fn map_in_range_lowest( - &self, - gem: &gem::Object, - object_range: Range, - alignment: u64, - range: Range, - prot: Prot, - guard_size: usize, - ) -> Result { - self.map_in_range_with_mode( - gem, - object_range, - alignment, - range, - prot, - guard_size, - mm::InsertMode::Low, - ) - } - - #[allow(clippy::too_many_arguments)] - fn map_in_range_with_mode( - &self, - gem: &gem::Object, - object_range: Range, - alignment: u64, - range: Range, - prot: Prot, - guard_size: usize, - mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -2453,30 +1941,29 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // End the bound-device lease before entering firmware/GPUVM waits. - let (_bind_lease, vm_bo) = { - let binding = self.device_binding.enter()?; - let dev = binding.device(); - let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let bo = self.gpuvm.obtain( - gem, - pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }), - )?; - prepare_sgt_segments(dev, gem, &bo, &lease)?; - (lease, bo) - }; + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } + let vm_bo = self + .gpuvm + .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; let size = object_range.range(); - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)? + self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM @@ -2497,7 +1984,7 @@ impl Vm { 0, range.start, range.end, - mode, + mm::InsertMode::Best, ); let mut node = match result { Ok(node) => node, @@ -2515,16 +2002,9 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. - let retain = ret.is_err() - && inner.uat_inner.m4_adopted - && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Map a GEM object into this Vm at a specific address. @@ -2542,32 +2022,33 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // End the bound-device lease before entering firmware/GPUVM waits. - let (_bind_lease, vm_bo) = { - let binding = self.device_binding.enter()?; - let dev = binding.device(); - let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let bo = self.gpuvm.obtain( - gem, - pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }), - )?; - prepare_sgt_segments(dev, gem, &bo, &lease)?; - (lease, bo) - }; + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } + let vm_bo = self + .gpuvm + .obtain(&gem, pin_init!(VmBoData { segments <-new_mutex!(None) }))?; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { self.gpuvm - .exec_lock_vm(/*Some(&gem)*/ None, false) + .exec_lock(/*Some(&gem)*/ None, false) .inspect_err(|_| { dev_err!(self.dev, "map_at(): Failed to exec_lock gpuvm\n"); })? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // Keep reservation-locking destructors out of the allocator payload @@ -2602,16 +2083,9 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. - let retain = ret.is_err() - && inner.uat_inner.m4_adopted - && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Map a range of a GEM object into this Vm using GPUVM. @@ -2629,16 +2103,14 @@ impl Vm { let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev: &device::Device = device.as_ref().as_ref(); - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } let vm_bo = self .gpuvm - .obtain(gem, pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; let mut resources = VmOpResources { preallocated_gpuvas: [ @@ -2651,13 +2123,19 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)? + self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + // Preallocate the page tables, to fail early if we ENOMEM inner.page_table.alloc_pages(addr..(addr + size))?; @@ -2700,22 +2178,6 @@ impl Vm { vm_exec.unique_gpuvm.sm_map(req) } - /// Spend the binding generation only for an incarnation exposed to firmware. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn mark_binding_published(&self) -> Result { - let uat = self.gpuvm.lock_inner().uat_inner.clone(); - let binding = self.binding.lock(); - let slot = binding.binding.as_ref().ok_or(EIO)?.slot() as usize + UAT_USER_CTX_START; - let mut shared = uat.lock(); - if shared.owned_user_roots[slot] != tagged_root(binding.ttb, slot) - || shared.ttbs()[slot].ttb0.load(Ordering::Acquire) != shared.owned_user_roots[slot] - { - return Err(EIO); - } - shared.binding_published[slot] = true; - Ok(()) - } - /// Add a direct MMIO mapping to this Vm at a free address. pub(crate) fn map_io( &self, @@ -2725,9 +2187,7 @@ impl Vm { size: usize, prot: Prot, ) -> Result { - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. - let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( @@ -2761,35 +2221,22 @@ impl Vm { let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; - let transaction_uat = inner.uat_inner.clone(); - let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); - let ret = inner - .page_table - .map_pages(iova..end, phys as PhysicalAddr, prot, false); - let discard = if ret.is_err() { - inner.page_table.discard_partial_map(iova..end) - } else { - Ok(()) - }; - drop(handoff); + let ret = inner.page_table.map_pages( + iova..end, + phys as PhysicalAddr, + prot, + false, + ); if ret.is_err() { - inner.finish_failed_map(iova, size); - } else { - inner.tlbi_contexts_naming_root(iova, size); - } - let retain = discard.is_err() && inner.uat_inner.m4_adopted; - if retain { - inner.uat_inner.failed.store(true, Ordering::Release); + inner.rollback_failed_map(iova, size as u64); } + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. drop(mutation); - drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } + let _ = inner; ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Unmap everything in an address range. @@ -2831,9 +2278,7 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. - let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); let _commit = if user { @@ -2864,7 +2309,7 @@ impl Vm { let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), result => result?, }; @@ -2891,66 +2336,6 @@ impl Vm { Ok(()) } - fn drop_mappings_before(&self, gem: &gem::Object, sequence: u64) -> Result { - let _residency = self.enter_retirement_residency(); - // Removing whole mappings only does unmaps, so no preallocated VAs - // are needed for the indexed path. - let mut resources = VmOpResources { - preallocated_gpuvas: [None, None, None], - }; - let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; - - // The BO's newest sequence is checked under the same lock as binds. If - // it is not newer than the close, every mapping belongs to that close - // and the per-object list is safe and substantially cheaper to walk. - // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { - Err(error) if error == ENOMEM => { - return self.drop_mappings_filtered(gem, sequence); - } - result => result?, - }; - let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) else { - return Ok(()); - }; - if bo.data().newest_bind.load(Ordering::Relaxed) > sequence { - drop(vm_exec); - drop(bo); - return self.drop_mappings_filtered(gem, sequence); - } - - mod_dev_dbg!(self.dev, "MMU: deferred bo_unmap\n"); - let result = vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo); - if result.is_ok() { - self.untrack_context_object_before(gem, sequence); - self.untrack_shared_object_before(gem, sequence); - } - drop(vm_exec); - drop(bo); - if result == Err(ENOMEM) { - return self.drop_mappings_filtered(gem, sequence); - } - result - } - - /// A newer binding shares the BO list, so select the closed generation by - /// walking the VM without allocating. This is also the ENOMEM fallback. - fn drop_mappings_filtered(&self, gem: &gem::Object, sequence: u64) -> Result { - let mut inner = self.gpuvm.lock_inner(); - let result = inner.unmap_object_where( - gem, - |va| va.data().bind_sequence <= sequence, - VmInner::unmap_gpuva, - ); - if result.is_ok() { - self.untrack_context_object_before(gem, sequence); - self.untrack_shared_object_before(gem, sequence); - } - drop(inner); - self.bo_deferred_cleanup(); - result - } - /// Preserve the indexed fast path normally; memory pressure must not lose /// accepted object cleanup just because an exec or operation list failed. fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result { @@ -2997,18 +2382,8 @@ impl Drop for VmInner { let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let mut transaction = match self.uat_inner.lock_handoff() { - Ok(guard) => guard, - Err(_) => { - self.page_table.retain(); - pr_err!( - "MMU: retaining M4 VM {} page tables after PPL failure\n", - self.id - ); - return; - } - }; - let uat_inner = transaction.shared_mut(); + let uat_inner = self.uat_inner.lock(); + uat_inner.handoff().lock(); let handoff_cur = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); let inval = ttb_cur == ttb; @@ -3021,10 +2396,9 @@ impl Drop for VmInner { } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); - uat_inner.owned_user_roots[idx] = 0; - uat_inner.publish_slot(idx); } - drop(transaction); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); // In principle we dropped all the KernelMappings already, but we might as // well play it safe and invalidate the whole ASID. @@ -3044,7 +2418,7 @@ impl Drop for VmInner { /// Physical regions of a UAT whose handoff the generation has already established itself /// ([`Uat::new_adopted`]). None of them needs to be in the kernel linear map. -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) struct AdoptedRegions { /// The handoff region (PPL magic, lock and flush requests). pub(crate) handoff: Range, @@ -3055,56 +2429,34 @@ pub(crate) struct AdoptedRegions { } impl Uat { - /// Common AP and firmware exclusion for G16's reserved-table window. Acquire - /// immutable root addresses before this guard, preserving binding -> UAT lock order. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn lock_m4_handoff(&self) -> Result> { - if !self.inner.m4_adopted { - return Err(EINVAL); - } - Ok(M4HandoffGuard { - _transaction: self.inner.lock_handoff()?, - }) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] /// Map a region given by its physical range, write-combined. - fn map_physical( - range: &Range, - min_size: usize, - reserved_ram: bool, - ) -> Result { + fn map_physical(range: &Range, min_size: usize) -> Result { let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; - if size < min_size || range.start & (UAT_PGMSK as PhysicalAddr) != 0 { + if size < min_size || (range.start | range.end) & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); } // SAFETY: All-zero is a valid `struct resource` (no name, parent or children). let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as usize; + raw.flags = kernel::bindings::IORESOURCE_MEM as _; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; - let flags = io::mem::MemFlag::WC; - #[cfg(CONFIG_DRM_ASAHI_G15)] - let flags = if reserved_ram { - // G15 admits only whole reserved RAM pages or wholly non-RAM carveouts. - if crate::g15::memory_mode(range.start, size as u64)? - == kernel::bindings::MEMREMAP_WB as u32 - { - io::mem::MemFlag::WB - } else { - flags - } + // A carveout outside System RAM is mapped write-combined; a reserved-memory region + // inside System RAM (mapped by the kernel) through the kernel's own mapping, as + // `map_region` does for the other generations. + let mode = crate::g15::memory_mode(range.start, size as u64)?; + let ram = mode == kernel::bindings::MEMREMAP_WB as u32; + let flags = if ram { + io::mem::MemFlag::WB } else { - flags + io::mem::MemFlag::WC }; - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - let _ = reserved_ram; - // SAFETY: The UAT regions are only accessed through this mapping by the CPU; - // the GPU reaches them through its own translation. + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU + // reaches them through its own translation. let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, @@ -3112,7 +2464,7 @@ impl Uat { }) } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] /// Creates a `Uat` for a handoff that the caller establishes itself (G15). /// /// This only maps the handoff and the TTBAT (write-combined); neither is written here. The @@ -3126,52 +2478,23 @@ impl Uat { /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after /// mapping and before the firmware uses the new addresses. Page tables are never freed on /// unmap, so linked entries stay valid. - #[allow(dead_code)] pub(crate) fn new_adopted( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, ) -> Result { - Self::new_adopted_inner(dev, geometry, regions, true, true) - } - - /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also - /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; - /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn new_adopted_with_user_upper( - dev: &driver::AsahiDrmDevice, - geometry: UatGeometry, - regions: &AdoptedRegions, - map_kernel_to_user: bool, - ) -> Result { - Self::new_adopted_inner(dev, geometry, regions, map_kernel_to_user, false) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - fn new_adopted_inner( - dev: &driver::AsahiDrmDevice, - geometry: UatGeometry, - regions: &AdoptedRegions, - map_kernel_to_user: bool, - reserved_ram: bool, - ) -> Result { - let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE, reserved_ram)?; - let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE, reserved_ram)?; + let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; + let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { firmware: UatFirmware::Handoff, - m4_adopted: !map_kernel_to_user, - failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, - reserved: None, - quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3180,14 +2503,10 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], - binding_generations: [0; UAT_NUM_CTX], - binding_published: [false; UAT_NUM_CTX], - owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, - map_kernel_to_user, + map_kernel_to_user: true, handoff_rgn, ttbs_rgn, - quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -3199,14 +2518,8 @@ impl Uat { let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let shadow_root = kernel::page::Page::into_phys(shadow); - let kernel_lower_vm = Vm::new( - dev.into(), - inner.clone(), - geometry.user_range(), - geometry, - None, - 1, - )?; + let kernel_lower_vm = + Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; let kernel_vm = Vm::new( dev.into(), inner.clone(), @@ -3233,13 +2546,13 @@ impl Uat { }) } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] /// Root of the kernel lower `Vm`, for TTBAT context 0 of an adopted handoff. pub(crate) fn kernel_lower_root(&self) -> PhysicalAddr { self.kernel_lower_vm.ttb() } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] /// Root whose driver-window entry names the second-level table of [`Self::kernel_vm`]'s /// driver window. With [`Self::new_adopted`] this is the shadow root that no context names. pub(crate) fn kernel_window_root(&self) -> PhysicalAddr { @@ -3303,16 +2616,13 @@ impl Uat { return Err(EINVAL); } let root = { - let owner = vm.gpuvm.lock_inner(); + let owner = lock_vm(&vm.gpuvm, None, false)?; if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { return Err(EINVAL); } owner.ttb() }; - Ok(self - .inner - .lock() - .contexts_naming_root(root, self.inner.m4_adopted)) + Ok(self.inner.lock().contexts_naming_root(root)) } /// Publish a completed alias set before its descriptors become visible. @@ -3341,24 +2651,12 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { - self.bind_inner(vm, true) - } - - /// Binds a VM without waiting for another slot owner to retire. - #[cfg(CONFIG_DRM_ASAHI_G15)] - pub(crate) fn try_bind(&self, vm: &Vm) -> Result { - self.bind_inner(vm, false) - } - - fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { - self.inner.check_m4()?; let mut binding = vm.binding.lock(); - self.inner.check_m4()?; if binding.binding.is_none() { assert_eq!(binding.active_users, 0); - let isolation = module_parameters::robust_isolation.value() != 0; + let isolation = *module_parameters::robust_isolation.value() != 0; self.slots.set_limit(if isolation { NonZeroUsize::new(1) @@ -3366,34 +2664,22 @@ impl Uat { None }); - let slot = if wait { - self.slots.get(binding.bind_token)? - } else { - self.slots.try_get(binding.bind_token)? - }; + let slot = self.slots.get(binding.bind_token)?; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let mut transaction = self.inner.lock_handoff()?; - let uat_inner = transaction.shared_mut(); - // On M4, empty bind/close churn must not wrap the wire generation - // back to firmware's last observed identity for a different root. - let previous = uat_inner.binding_generations[idx]; - let incarnation = if !self.inner.m4_adopted - || previous == 0 - || uat_inner.binding_published[idx] - { - previous.checked_add(1).ok_or(EOVERFLOW)? + let uat_inner = self.inner.lock(); + + let ttb1 = if uat_inner.map_kernel_to_user { + uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT } else { - previous + 0 }; - let ttb1 = - application_ttb1(uat_inner.kernel_ttb1, idx, uat_inner.map_kernel_to_user); - let ttbs = uat_inner.ttbs(); + uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { pr_err!( "Vm::bind to slot {}, but it is currently in use by the ASC?\n", @@ -3402,12 +2688,8 @@ impl Uat { } ttbs[idx].ttb0.store(ttb, Ordering::Release); ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.binding_generations[idx] = incarnation; - uat_inner.binding_published[idx] = false; - uat_inner.owned_user_roots[idx] = ttb; - binding.generation = binding_wire_generation(incarnation); - uat_inner.publish_slot(idx); - drop(transaction); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); // Make sure all TLB entries from the previous owner of this ASID are gone mem::sync(); @@ -3426,35 +2708,6 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } - /// Confirms the exact stopped-adopted M4 TTBAT contract. The same mutex serializes every - /// host root installation/removal; firmware does not write these words. An unexpected - /// observed root is an error, never a new ownership grant. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn check_adopted_bindings(&self) -> Result { - self.inner.check_m4()?; - // Binding locks precede the global UAT lock, as in bind(). - let lower = self.kernel_lower_vm.ttb() | TTBR_VALID; - let inner = self.inner.lock(); - self.inner.check_m4()?; - if self.inner.firmware != UatFirmware::Handoff || inner.map_kernel_to_user { - return Err(EINVAL); - } - let ttbs = inner.ttbs(); - if ttbs[0].ttb0.load(Ordering::Acquire) != lower - || ttbs[0].ttb1.load(Ordering::Acquire) != inner.kernel_ttb1 | TTBR_VALID - { - return Err(EIO); - } - for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { - if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] - || table.ttb1.load(Ordering::Acquire) != 0 - { - return Err(EIO); - } - } - Ok(()) - } - /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { @@ -3462,7 +2715,7 @@ impl Uat { return Err(EINVAL); } { - let owner = vm.gpuvm.lock_inner(); + let owner = lock_vm(&vm.gpuvm, None, false)?; if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) { @@ -3506,7 +2759,6 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); - inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -3516,17 +2768,14 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - self.inner.check_m4()?; - let vm = Vm::new( + Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, self.geometry, None, id, - )?; - self.inner.check_m4()?; - Ok(vm) + ) } /// Returns the address space geometry of this UAT. @@ -3548,19 +2797,14 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries( - self.inner.reserved.as_deref(), - self.kernel_vm.ttb(), - UAT_PGSZ / size_of::(), - |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }, - ) + pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }) }?; mem::sync(); @@ -3573,16 +2817,11 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries( - self.inner.reserved.as_deref(), - self.kernel_vm.ttb(), - snapshot.len(), - |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }, - ) + pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }) }?; if !matches { return Err(EIO); @@ -3594,19 +2833,15 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let Some(secondary) = self.inner.lock().secondary_ttb1 else { - // A single-instance firmware has no secondary root to mirror. - return Ok(()); - }; + let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; - let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(reserved, secondary, count, |secondary| { + pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -3623,33 +2858,24 @@ impl Uat { Ok(()) } - /// Discards the CPU's cached lines of the no-map table regions after the firmware built its - /// root there during boot (`UatQuirks::reserved_tables`; see `ReservedTables::invalidate`). - pub(crate) fn invalidate_reserved_tables(&self) { - if let Some(reserved) = self.inner.reserved.as_ref() { - reserved.invalidate(); - } - } - /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff_lock(); + inner.handoff().lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff_unlock(); + inner.handoff().unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.publish_slot(KERNEL_ALIAS_CTX); - inner.handoff_unlock(); + inner.handoff().unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -3659,30 +2885,21 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner( - dev: driver::AsahiDevice, - firmware: UatFirmware, - quirks: UatQuirks, - reserved: Option>, - ) -> Result> { + fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); - let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, - m4_adopted: false, - failed: AtomicBool::new(false), - clean_poc: quirks.clean_tables, - quirks, - reserved, + #[cfg(CONFIG_DRM_ASAHI_G15)] + clean_poc: false, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3691,14 +2908,10 @@ impl Uat { kernel_ttb1: 0, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], - binding_generations: [0; UAT_NUM_CTX], - binding_published: [false; UAT_NUM_CTX], - owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, - quirks: shared_quirks, }, "uat_shared" ), @@ -3714,27 +2927,12 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, - quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - mem::set_page_tlbi(quirks.page_tlbi); - let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; - let reserved = if quirks.reserved_tables { - let mut tables = pgtable::ReservedTables::new(); - for name in [c"pagetables", c"l2-pagetables"] { - let res = of_node.reserved_mem_region_to_resource_byname(name)?; - // SAFETY: The boot loader reserved these regions (no-map) for the firmware's - // page tables; they stay reserved until reboot. - unsafe { tables.add(res)? }; - } - Some(Arc::new(tables, GFP_KERNEL)?) - } else { - None - }; - - let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let inner = Self::make_inner(dev.clone(), firmware)?; + let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -3742,11 +2940,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset: Some(offset), - } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), - UatFirmware::PublishedRoots { - secondary_root_offset: None, - } => (UAT_PGSZ, None), + secondary_root_offset, + } => ( + secondary_root_offset as usize + UAT_PGSZ, + Some(ttb1 + secondary_root_offset), + ), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -3797,18 +2995,11 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware, quirks.handoff_slot)?; - if quirks.clean_tables { - pgtable::clean_poc( - core::ptr::from_ref(inner.handoff()).cast::(), - HANDOFF_SIZE, - ); - pgtable::poc_barrier(); - } + inner.handoff().init(firmware)?; dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff_lock(); + inner.handoff().lock(); let ttbs = inner.ttbs(); @@ -3820,7 +3011,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if matches!(firmware, UatFirmware::PublishedRoots { .. }) { + if secondary_ttb1.is_some() { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3829,12 +3020,8 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } - if quirks.clean_tables { - pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); - pgtable::poc_barrier(); - } - inner.handoff_unlock(); + inner.handoff().unlock(); core::mem::drop(inner); @@ -3852,30 +3039,3 @@ impl Drop for Uat { mem::sync(); } } - -impl Vm { - /// Diagnostic reservation occupancy (including guards), not a placement promise. - /// Removal can progress between queries; allocations are excluded by the VM lock. - pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { - let inner = self.gpuvm.lock_inner(); - let mut next = range.start; - let mut used = 0; - let mut largest = 0; - while next < range.end { - let Some(node) = inner.mm.first_overlap(next..range.end)? else { - largest = largest.max(range.end - next); - break; - }; - largest = largest.max(node.start.saturating_sub(next)); - let end = node.end.min(range.end); - used += end - node.start.max(next); - next = end; - } - Ok((used, largest)) - } -} -impl Uat { - pub(crate) fn execution_contexts_used(&self) -> u32 { - (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() - } -} diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 73896fea53e00..5f7e572a3a462 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only +// SPDX-License-Identifier: GPL-2.0-only OR MIT //! Support for Apple RTKit coprocessors. //! @@ -222,7 +222,6 @@ impl RtKit { } else { 0 }, - syslog_is_debug: None, crashed: Some(crashed_callback::), recv_message: Some(recv_message_callback::), recv_message_early: Some(recv_message_early_callback::), @@ -231,18 +230,11 @@ impl RtKit { } else { None }, - shmem_setup_ep: None, shmem_destroy: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { Some(shmem_destroy_callback::) } else { None }, - protocol_version: 0, - }; - - const COREANALYTICS_VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { - flags: Self::VTABLE.flags | bindings::APPLE_RTKIT_NEEDS_COREANALYTICS, - ..Self::VTABLE }; /// Creates a new RTKit client for a given device and optional mailbox name or index. @@ -251,29 +243,6 @@ impl RtKit { mbox_name: Option<&'static CStr>, mbox_idx: usize, data: T::Data, - ) -> Result { - Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::VTABLE) - } - - /// Creates a client that requires CoreAnalytics setup during boot. - /// - /// The caller must qualify this dependency for the particular firmware - /// instance. This selects immutable operations before mailbox RX starts. - pub fn new_with_coreanalytics( - dev: &device::Device, - mbox_name: Option<&'static CStr>, - mbox_idx: usize, - data: T::Data, - ) -> Result { - Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::COREANALYTICS_VTABLE) - } - - fn new_with_ops( - dev: &device::Device, - mbox_name: Option<&'static CStr>, - mbox_idx: usize, - data: T::Data, - ops: &'static bindings::apple_rtkit_ops, ) -> Result { let ptr: *mut crate::ffi::c_void = data.into_foreign().cast(); let guard = ScopeGuard::new(|| { @@ -291,7 +260,7 @@ impl RtKit { None => ptr::null(), }, mbox_idx.try_into()?, - ops, + &Self::VTABLE, )) }?; @@ -305,10 +274,6 @@ impl RtKit { }) } - /// Borrows the C handle without transferring ownership. - /// The owner must outlive each unsafe C operation and its callbacks. - pub fn as_raw(&self) -> *mut bindings::apple_rtkit { self.rtk } - /// Requests acknowledged RTKit shutdown using the C core's bounded waits. /// /// The caller must not hold a lock needed by its receive callbacks: the @@ -357,57 +322,6 @@ impl RtKit { pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } } - - /// Sets the minimum utilization (0..=1024) the coprocessor's receive thread runs with, or - /// restores the scheduler's default with `None`. - /// - /// The receive thread applies it to itself when it next runs. Returns `EOPNOTSUPP` for a - /// mailbox without a threaded receive handler. - pub fn set_rx_util_min(&self, util_min: Option) -> Result { - // INVARIANT: `rtk` is valid per the type invariant. - RxUtilMin { rtk: self.rtk }.set(util_min) - } - - /// Returns a handle that sets the receive thread's minimum utilization (see - /// [`RtKit::set_rx_util_min`]) without borrowing this [`RtKit`], from any context. - /// - /// # Safety - /// - /// The caller must not use the handle after this [`RtKit`] is dropped. - pub unsafe fn rx_util_min_handle(&self) -> RxUtilMin { - // INVARIANT: The caller keeps this `RtKit`, and so `rtk`, alive while using the handle. - RxUtilMin { rtk: self.rtk } - } -} - -/// Sets the minimum utilization of an RTKit coprocessor's receive thread. -/// -/// The request is a single store that the receive thread applies to itself when it next runs, -/// so it may be made from any context. -/// -/// # Invariants -/// -/// `rtk` is valid while the handle is used. -#[derive(Clone, Copy)] -pub struct RxUtilMin { - rtk: *mut bindings::apple_rtkit, -} - -// SAFETY: The request is one store into the mailbox, valid from any thread. -unsafe impl Send for RxUtilMin {} -// SAFETY: As above; the handle has no state of its own. -unsafe impl Sync for RxUtilMin {} - -impl RxUtilMin { - /// See [`RtKit::set_rx_util_min`]. - pub fn set(&self, util_min: Option) -> Result { - let util_min = match util_min { - Some(v) => i32::try_from(v).map_err(|_| EINVAL)?, - None => -1, - }; - // SAFETY: `rtk` is valid per the type invariant; the call only stores the request. - to_result(unsafe { bindings::apple_rtkit_set_rx_util_min(self.rtk, util_min) }) - } } // SAFETY: `RtKit` operations require a mutable reference From e45af192f61d218a59fc2ac06514d1bdd6a6e267 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 01:11:35 -0400 Subject: [PATCH 03/37] drm/asahi: m3: start the T8122 runtime on 7.2 Wire the restored 7.1.12 M3 runtime to the 7.2 driver behind CONFIG_DRM_ASAHI_M3_T8122 (default n), so the M3 MacBook Air (J613, J615) GPU runs again on its 14.8.3 firmware. A T8122 probe still passes the G15G identity gate and the backend selection first, and the runtime still starts only with asahi.t8122_start=1; otherwise the probe is refused as before. Other GPUs are not affected: T6030/T6031 keep the G15 backend or the refusal, and the T6030/T6031 start paths and the T6031 start experiment are dropped from the runtime. On 7.2 the runtime: - registers through drm::Registration as a gpu::Gpu backend, like G15 and G17, and is stopped on unbind while its device resources are live; the completion worker moves from the DRM device data into the runtime's RTKit state; - keeps user mappings alive across accepted work with the shared VM job lifetime (VmJobGuard, retain_job, deferred unmaps), the generalized form of 7.1.12's M3 VM job lifetime, on VMs created with tracked status; - adopts the firmware's UAT as the G15 backend does on 7.2 (Uat::new_adopted and g15::uat: the PPL exchange, TTBAT context 0, the host window at root entry 2 through the shared L2) instead of 7.1.12's M3-only handoff mode in mmu.rs/pgtable.rs. The firmware- walked pages are cleaned to the point of coherency, the host window is linked before the InitData and each batch reach the firmware, and context 0 is invalidated when a job's command aliases go. The option therefore depends on DRM_ASAHI_G15; - refuses VMs to userspace that did not ask GET_PARAMS for the whole global parameter structure (7.1.12's per-file client gate), through a new gpu::Gpu::admit_vm hook that admits every client elsewhere. Smaller 7.2 adaptations: the IoMem/Devres, IoSysMap and RTKit buffer APIs; Queue::submit's registration argument; quiet MMIO mappings in an mmu::m3 helper module; the M3 debug flags, F32::to_bits, Completion::complete/try_wait_for_completion/wait_for_completion_timeout and RtKit::is_running/is_crashed, which 7.2 dropped; the M3 module parameters move into m3_params, and m3_soc.rs gets back its full 7.1.12 table, cfg-gated with the runtime. asahi.m3_expose=0 is refused before anything starts. The devcoredump fault snapshot is dropped (7.2's kernel crate has no devcoredump binding): the first failure is logged. Generated-by: LLM Co-Authored-By: Claude Opus 5.5 Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/Kconfig | 15 + drivers/gpu/drm/asahi/debug.rs | 7 + drivers/gpu/drm/asahi/driver.rs | 57 ++++ drivers/gpu/drm/asahi/file.rs | 14 + drivers/gpu/drm/asahi/float.rs | 5 + drivers/gpu/drm/asahi/g15/mod.rs | 4 +- drivers/gpu/drm/asahi/g15/regs.rs | 15 + drivers/gpu/drm/asahi/g15/uat.rs | 33 +- drivers/gpu/drm/asahi/gpu.rs | 6 + drivers/gpu/drm/asahi/m3/agx_fault.rs | 87 ----- drivers/gpu/drm/asahi/m3/g15_raw.rs | 30 +- drivers/gpu/drm/asahi/m3/hw.rs | 18 +- drivers/gpu/drm/asahi/m3/hw/agx3.rs | 2 +- drivers/gpu/drm/asahi/m3/initdata.rs | 346 +------------------- drivers/gpu/drm/asahi/m3/m3_adt_config.rs | 146 +++------ drivers/gpu/drm/asahi/m3/m3_board.rs | 27 +- drivers/gpu/drm/asahi/m3/m3_client.rs | 40 +-- drivers/gpu/drm/asahi/m3/m3_completion.rs | 8 +- drivers/gpu/drm/asahi/m3/m3_compute.rs | 49 +-- drivers/gpu/drm/asahi/m3/m3_config.rs | 123 +++----- drivers/gpu/drm/asahi/m3/m3_device.rs | 117 +++---- drivers/gpu/drm/asahi/m3/m3_drm.rs | 197 +++++++----- drivers/gpu/drm/asahi/m3/m3_firmware.rs | 57 +--- drivers/gpu/drm/asahi/m3/m3_memory.rs | 54 +++- drivers/gpu/drm/asahi/m3/m3_params.rs | 341 ++++---------------- drivers/gpu/drm/asahi/m3/m3_pass.rs | 6 +- drivers/gpu/drm/asahi/m3/m3_pool.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_render.rs | 73 ++--- drivers/gpu/drm/asahi/m3/m3_resources.rs | 12 +- drivers/gpu/drm/asahi/m3/m3_rtkit.rs | 46 +-- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 288 +++++++++-------- drivers/gpu/drm/asahi/m3/m3_soc.rs | 367 +++++++++++++++++++--- drivers/gpu/drm/asahi/m3/m3_submit.rs | 48 +-- drivers/gpu/drm/asahi/m3/m3_thermal.rs | 17 +- drivers/gpu/drm/asahi/m3/mod.rs | 94 +++++- drivers/gpu/drm/asahi/m3/t8122_knobs.rs | 2 +- drivers/gpu/drm/asahi/m3/t8122_start.rs | 22 +- drivers/gpu/drm/asahi/mmu.rs | 2 + drivers/gpu/drm/asahi/mmu/m3.rs | 142 +++++++++ rust/kernel/soc/apple/rtkit.rs | 12 + rust/kernel/sync/completion.rs | 22 ++ 41 files changed, 1467 insertions(+), 1486 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/m3/agx_fault.rs create mode 100644 drivers/gpu/drm/asahi/mmu/m3.rs diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 958600a91b06b..4cfe26f28b891 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -36,6 +36,21 @@ config DRM_ASAHI_DEBUG_ALLOCATOR Say N unless you are debugging firmware structures or porting to a new firmware version. +config DRM_ASAHI_M3_T8122 + bool "M3 MacBook Air GPU runtime (T8122, 14.8.3 firmware)" + depends on DRM_ASAHI + # The runtime adopts the firmware's UAT through the G15 backend's handoff helpers. + depends on DRM_ASAHI_G15 + select CRYPTO_LIB_SHA256 + default n + help + Build the M3 runtime that starts the GPU of the M3 MacBook Air + (J613, J615) on the 14.8.3 GPU firmware the boot loader leaves + loaded. It starts only with asahi.t8122_start=1 on the kernel + command line, after the boot loader handed the GPU over; without + it the T8122 probe is refused as before. Other GPUs are not + affected. + config DRM_ASAHI_G15 bool "Native G15 firmware backend (M3 Pro/Max)" depends on DRM_ASAHI diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs index 50628ade5ab8e..e8087442d5320 100644 --- a/drivers/gpu/drm/asahi/debug.rs +++ b/drivers/gpu/drm/asahi/debug.rs @@ -54,6 +54,13 @@ pub(crate) enum DebugFlags { DeviceControlCh = 21, FwCtlCh = 22, + // Host submission/completion timestamps without verbose firmware tracing. + SubmitTiming = 23, + // Low-volume M3 command/geometry totals without per-IRQ printk. + M3SubmitSummary = 24, + // Per-slot retired timestamps, without enabling firmware ktrace. + M3PassTiming = 25, + // 32-35: Allocator debugging FillAllocations = 32, DebugAllocations = 33, diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index c092595180cf8..43fa17412815d 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -49,6 +49,8 @@ pub(crate) struct AsahiPlatformDriverData<'bound> { _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, + /// The M3 T8122 runtime, stopped on unbind like the G17 GPU. + m3: Option, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. raw_dev: *mut bindings::device, @@ -62,6 +64,11 @@ unsafe impl Send for AsahiPlatformDriverData<'_> {} // SAFETY: As above. unsafe impl Sync for AsahiPlatformDriverData<'_> {} +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +type M3Backend = Arc; +#[cfg(not(CONFIG_DRM_ASAHI_M3_T8122))] +type M3Backend = core::convert::Infallible; + /// Holds a reference to the top-level GPU object. #[pin_data] pub(crate) struct AsahiDrmRegistrationData<'drm> { @@ -213,6 +220,11 @@ impl platform::Driver for AsahiPlatformDriver { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + if let Some(m3) = this.m3.as_ref() { + // As for G17: the runtime stops its coprocessor through device resources. + m3.stop(); + } } /// Device probe function. @@ -227,6 +239,8 @@ impl platform::Driver for AsahiPlatformDriver { let cfg = match info.ok_or(ENODEV)? { ProbeConfig::Legacy(cfg) => *cfg, ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg), + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + ProbeConfig::M3(crate::m3::Chip::T8122) => return AsahiPlatformDriver::probe_t8122(pdev), ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; @@ -297,6 +311,7 @@ impl platform::Driver for AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + m3: None, raw_dev, raw_stats, stats, @@ -328,6 +343,43 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + m3: None, + raw_dev: core::ptr::null_mut(), + raw_stats: core::ptr::null_mut(), + stats: None, + }) + } + + /// The M3 MacBook Air (T8122): the M3 runtime, started only when its identity gate, the + /// backend selection and the start experiment (`asahi.t8122_start=1`) admit it; otherwise + /// the probe is refused as without the runtime. + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + fn probe_t8122<'bound>( + pdev: &'bound platform::Device>, + ) -> Result> { + let soc = crate::m3::t8122_admitted(pdev)?; + // SAFETY: The admitted M3 GPU's UAT outputs 42-bit physical addresses. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; + let resources = regs::Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?; + let gpu = backend.clone() as Arc; + let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); + // SAFETY: The registration is stored and unregistered by platform data destruction, + // which drops the runtime with it. + let reg = match unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0) } { + Ok(reg) => reg, + Err(error) => { + backend.stop(); + return Err(error); + } + }; + dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); + Ok(AsahiPlatformDriverData { + _device: reg.device().into(), + _reg: reg, + g17: None, + m3: Some(backend), raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -371,6 +423,7 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: Some(gpu), + m3: None, raw_dev, raw_stats, stats: None, @@ -390,5 +443,9 @@ impl Drop for AsahiPlatformDriverData<'_> { // Also covers failure to allocate platform driver data after the probe body returned. gpu.shutdown(); } + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + if let Some(m3) = self.m3.as_ref() { + m3.stop(); + } } } diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 608864949c976..242f6cdcd3f83 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -233,6 +233,9 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, + /// Largest size this file has asked GET_PARAMS for the global parameters with + /// ([`gpu::Gpu::admit_vm`]). + params_size: core::sync::atomic::AtomicU64, } /// Convenience type alias for our DRM `File` type. @@ -286,6 +289,7 @@ impl File { (*slot).id = id; (*slot).user_range = user_range; + addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); Ok(()) }) } @@ -386,6 +390,9 @@ impl File { core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) })?; + file.inner() + .params_size + .fetch_max(data.size, core::sync::atomic::Ordering::Relaxed); Ok(0) } @@ -454,6 +461,13 @@ impl File { // Every other runtime here reports DRM_ASAHI_GPU_HAL_LEGACY. return Err(EINVAL); } + reg_data.gpu.admit_vm( + device.as_ref().as_ref(), + file.inner() + .params_size + .load(core::sync::atomic::Ordering::Relaxed), + data.flags, + )?; let kernel_range = data.kernel_start..data.kernel_end; let user_range = file.inner().user_range.clone(); diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 46c05699d7722..a740ad3bb945d 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -49,6 +49,11 @@ impl F32 { F32(u) } + /// Convert to the raw IEEE 754 bits. + pub(crate) const fn to_bits(self) -> u32 { + self.0 + } + // Convert a `f32` value into an F32 // // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 75d3ecc30ffe5..324bc08194e53 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -68,10 +68,10 @@ mod pools; mod power; mod queue; mod recovery; -mod regs; +pub(crate) mod regs; mod slots; mod timestamp; -mod uat; +pub(crate) mod uat; mod vm; mod wire; diff --git a/drivers/gpu/drm/asahi/g15/regs.rs b/drivers/gpu/drm/asahi/g15/regs.rs index d5435e9f1c7ea..05a1b390b3dff 100644 --- a/drivers/gpu/drm/asahi/g15/regs.rs +++ b/drivers/gpu/drm/asahi/g15/regs.rs @@ -152,6 +152,21 @@ impl Mapping { pub(crate) fn phys(&self) -> u64 { self.phys } + + /// Cleans the whole mapping to the point of coherency when it is the kernel's cacheable + /// mapping of reserved System RAM ([`Self::region`]); device and write-combined mappings + /// need nothing. + pub(crate) fn clean_poc(&self) { + if self.kind != Kind::WriteBack { + return; + } + let start = self.raw.addr(); + // SAFETY: `start..start + maxsize` is this live mapping; cleaning a cacheable range has + // no other effect. + unsafe { + bindings::dcache_clean_poc(start as _, (start + self.raw.size()) as _); + } + } } impl Drop for Mapping { diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 854ff34e729a3..3855bd0beb9cb 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -200,16 +200,41 @@ pub(crate) struct Window { impl Window { /// Maps the UAT handoff regions ([`Mapping::region`]). pub(crate) fn new(adt: &super::adt::GpuAdt) -> Result { + Self::from_regions(adt.handoff.start, adt.ttbs.start, adt.tables.start, adt.shared_l2.start) + } + + /// Maps the UAT handoff regions at the given physical addresses: the handoff page, the + /// TTBAT, the firmware kernel root and the shared L2 page. The M3 T8122 runtime takes them + /// from its admitted reserved-memory regions. + pub(crate) fn from_regions(handoff: u64, ttbs: u64, root: u64, shared_l2: u64) -> Result { Ok(Self { - handoff: Mapping::region(adt.handoff.start, hw::HANDOFF_SIZE as usize)?, - ttbs: Mapping::region(adt.ttbs.start, hw::TTBS_SIZE as usize)?, - root: Mapping::region(adt.tables.start, hw::PAGE as usize)?, - shared_l2: Mapping::region(adt.shared_l2.start, hw::SHARED_L2_SIZE as usize)?, + handoff: Mapping::region(handoff, hw::HANDOFF_SIZE as usize)?, + ttbs: Mapping::region(ttbs, hw::TTBS_SIZE as usize)?, + root: Mapping::region(root, hw::PAGE as usize)?, + shared_l2: Mapping::region(shared_l2, hw::SHARED_L2_SIZE as usize)?, image: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?, snapshot: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?, }) } + /// [`Self::sync_range`] over the whole host window, then [`Self::clean_tables`]. + pub(crate) fn sync_all(&mut self, uat: &mmu::Uat) -> Result { + self.sync_inner(uat, None)?; + self.clean_tables(); + Ok(()) + } + + /// Cleans the firmware-walked pages this window writes (the kernel root, the shared L2, + /// the TTBAT and the handoff page) to the point of coherency. Only a region that is + /// reserved System RAM, mapped cacheable, has anything to clean: the UAT walker does not + /// snoop the CPU caches. The M3 T8122 runtime calls it after [`Self::bootstrap`]. + pub(crate) fn clean_tables(&self) { + for map in [&self.root, &self.shared_l2, &self.ttbs, &self.handoff] { + map.clean_poc(); + } + mb(); + } + /// Cold-start check: all 128 TTBAT words are zero. pub(crate) fn ttbat_is_clear(&self) -> Result { let t = self.ttbs.io(); diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 90bb25451019e..b9283efb470a9 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -338,6 +338,12 @@ pub(crate) trait Gpu: Send + Sync { } /// A userspace VM handle was closed; release backend-only registry references. fn vm_closed(&self) {} + /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS + /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 + /// T8122 runtime admits any client. + fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { + Ok(()) + } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; /// Create a new user command queue. diff --git a/drivers/gpu/drm/asahi/m3/agx_fault.rs b/drivers/gpu/drm/asahi/m3/agx_fault.rs deleted file mode 100644 index 9c37cfa98dce9..0000000000000 --- a/drivers/gpu/drm/asahi/m3/agx_fault.rs +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Immutable fault-only snapshots of owned M3 firmware memory. -//! -//! Records contain name[32], firmware VA, length, begin/end monotonic ns, -//! then exactly `length` bytes. No MMIO or application BOs are read. Reads -//! are sequential observations, not atomic with running firmware. - -use kernel::{prelude::*, time::{ClockSource, Monotonic}}; - -pub(crate) struct Dump { - bytes: KVVec, - count: u32, - limit: usize, -} - -impl Dump { - - /// M3 snapshots have no scheduler job ID yet and contain only bounded - /// driver-owned firmware configuration and channel storage. - pub(crate) fn new_m3() -> Result { - Self::with_format(0, b"M3FWD001", 2 * 1024 * 1024) - } - - fn with_format(job: u64, magic: &[u8; 8], limit: usize) -> Result { - // Reserve before firmware/job publication, while healthy. Record appends - // must never enter reclaim waiting on the GPU that just faulted. - let mut bytes = KVVec::with_capacity(limit, GFP_KERNEL)?; - bytes.resize(40, 0, GFP_KERNEL)?; - bytes[..8].copy_from_slice(magic); - bytes[16..24].copy_from_slice(&job.to_le_bytes()); - bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); - Ok(Self { bytes, count: 0, limit }) - } - - /// Start the one-shot capture; construction may precede the fault by hours. - pub(crate) fn begin(&mut self, job: u64) { - self.bytes[16..24].copy_from_slice(&job.to_le_bytes()); - self.bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); - } - - pub(crate) fn record(&mut self, name: &str, va: u64, size: usize, - read: impl FnOnce(&mut [u8]) -> Result) -> Result { - if name.is_empty() || name.len() >= 32 || size == 0 { return Err(EINVAL); } - let count = self.count.checked_add(1).ok_or(EOVERFLOW)?; - let start = self.bytes.len(); - let end = start.checked_add(64).and_then(|x| x.checked_add(size)).ok_or(EOVERFLOW)?; - if end > self.limit { return Err(E2BIG); } - // Capacity was reserved at construction. NOWAIT is defense in depth; - // the checked limit above guarantees resize cannot allocate here. - self.bytes.resize(end, 0, GFP_NOWAIT)?; - self.bytes[start..start+name.len()].copy_from_slice(name.as_bytes()); - self.bytes[start+32..start+40].copy_from_slice(&va.to_le_bytes()); - self.bytes[start+40..start+48].copy_from_slice(&(size as u64).to_le_bytes()); - let begin = Monotonic::ktime_get() as u64; - if let Err(error) = read(&mut self.bytes[start+64..end]) { - self.bytes.truncate(start); - return Err(error); - } - let finish = Monotonic::ktime_get() as u64; - self.bytes[start+48..start+56].copy_from_slice(&begin.to_le_bytes()); - self.bytes[start+56..start+64].copy_from_slice(&finish.to_le_bytes()); - self.count = count; - Ok(()) - } - - pub(crate) fn publish(mut self, dev: &kernel::device::Device) -> Result { - let length = self.bytes.len(); - self.bytes[8..12].copy_from_slice(&self.count.to_le_bytes()); - self.bytes[12..16].copy_from_slice(&1u32.to_le_bytes()); - self.bytes[32..40].copy_from_slice(&(length as u64).to_le_bytes()); - let format = "M3FWD001"; - let owned = KBox::new(self, GFP_NOWAIT)?; - kernel::devcoredump::dev_coredump(dev, &crate::THIS_MODULE, owned, GFP_NOWAIT, - kernel::devcoredump::DEFAULT_TIMEOUT); - dev_info!(dev, "Asahi: sealed {}-byte firmware fault snapshot offered to devcoredump ({})\n", length, format); - Ok(()) - } -} - -impl kernel::devcoredump::DevCoreDump for Dump { - fn read(&self, output: &mut [u8], offset: usize) -> Result { - if offset >= self.bytes.len() { return Ok(0); } - let length = output.len().min(self.bytes.len()-offset); - output[..length].copy_from_slice(&self.bytes[offset..offset+length]); - Ok(length) - } -} diff --git a/drivers/gpu/drm/asahi/m3/g15_raw.rs b/drivers/gpu/drm/asahi/m3/g15_raw.rs index a876cad1ec283..785b7f2314c8f 100644 --- a/drivers/gpu/drm/asahi/m3/g15_raw.rs +++ b/drivers/gpu/drm/asahi/m3/g15_raw.rs @@ -2,8 +2,8 @@ //! GPU initialization / global structures -use super::channels; -use super::types::*; +use crate::fw::channels; +use crate::fw::types::*; use crate::{ default_zeroed, gem, @@ -1158,7 +1158,9 @@ pub(crate) mod raw { pub(crate) event: ChannelRing, pub(crate) fw_log: ChannelRing, pub(crate) ktrace: ChannelRing, - pub(crate) stats: ChannelRing, + // The G15 stats record has the 13.x size (STATS_SZ for V >= V13_0B4); 7.2 generates + // only the 13.x instances. + pub(crate) stats: ChannelRing, /// FWLog payload buffer (0x600 x 0xd8). +0x1f8 @@ -1548,7 +1550,7 @@ impl GpuStruct for InitDataG15V14_8_3 { // G15 / 14.8.3 firmware structure layout checks: the sizes and offsets the firmware expects. mod g15_layout { - use super::channels; + use crate::fw::channels; use super::raw::*; use core::mem::{offset_of, size_of}; use kernel::static_assert; @@ -1602,7 +1604,7 @@ mod g15_layout { static_assert!(size_of::() == 0xe00); static_assert!(size_of::() == 0x18); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_28) == 0x28); - static_assert!(offset_of!(GlobalsG15V14_8_3, debug) == crate::m3_globals_debug::OFFSET); + static_assert!(offset_of!(GlobalsG15V14_8_3, debug) == crate::m3::m3_globals_debug::OFFSET); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_38) == 0x38); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_50) == 0x50); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_56) == 0x56); @@ -1778,23 +1780,5 @@ mod g15_layout { == 0xc30 ); - // G13/G14X 13.5 sizes must not change (regression guard for the per-field gating). - // RuntimePointers: gpu_scratch at 0x280 (0x270 on G14X) + RuntimeScratch; note that these - // are 0x20 more than ver_info word 0 (0x6ba0/0xb390). The others are the current G13/G14X - // sizes. - static_assert!(size_of::>() == 0x6bc0); - static_assert!(size_of::>() == 0xb3b0); - static_assert!(size_of::() == 0x12394); - static_assert!(size_of::() == 0x1715c); - static_assert!(size_of::() == 0x421c); - static_assert!(size_of::() == 0x6c34); - static_assert!(size_of::() == 0x1884); - static_assert!(size_of::() == 0x1884); - static_assert!(size_of::>() == 0xbc); - static_assert!(size_of::>() == 0xbc); - static_assert!(size_of::>() == 0xb4); - static_assert!(size_of::() == 0x30); - static_assert!(size_of::() == 0x1124); - static_assert!(size_of::() == 0x1a34); static_assert!(size_of::>() == 0x30); } diff --git a/drivers/gpu/drm/asahi/m3/hw.rs b/drivers/gpu/drm/asahi/m3/hw.rs index c90156bd1244b..a774f5de6573f 100644 --- a/drivers/gpu/drm/asahi/m3/hw.rs +++ b/drivers/gpu/drm/asahi/m3/hw.rs @@ -5,21 +5,18 @@ //! This module contains the definitions used to store per-GPU and per-SoC configuration data. use crate::fw::types::*; -pub(crate) use crate::identity::{GpuGen, GpuHalGeneration, GpuVariant}; +pub(crate) use crate::m3::identity::{GpuGen, GpuHalGeneration, GpuVariant}; use kernel::c_str; use kernel::prelude::*; const MAX_POWERZONES: usize = 5; +/// G15 SRAM rail floor for a device tree without `apple,min-sram-microvolt`, in microvolts: the +/// J516S runtime ADT perf-states-sram table floors SRAM at 810 mV (7.1.12 `hw::t6030`). +const G15_MIN_SRAM_MICROVOLT: u32 = 810_000; + pub(crate) mod agx3; -pub(crate) mod t600x; -pub(crate) mod t602x; -pub(crate) mod t6030; -pub(crate) mod t6031; -pub(crate) mod t8103; -pub(crate) mod t8112; pub(crate) mod t8122; -pub(crate) mod t8140; /// GPU revision enumeration. Note: Part of the UABI. #[derive(Debug, PartialEq, Copy, Clone)] @@ -559,6 +556,9 @@ impl PwrConfig { return Err(EINVAL); } + // Kept as on the 7.1.12 kernel this runtime was qualified with: it steps by 3 over the + // zone count, which reads the same zones as one zone per triple for up to one zone and + // fewer beyond. (7.2's shared copy reads one zone per triple.) let pz_count = pz_data.len() / 3; let mut power_zones = KVec::new(); for i in (0..pz_count).step_by(3) { @@ -610,7 +610,7 @@ impl PwrConfig { min_sram_microvolt: if cfg.gpu_gen == GpuGen::G15 { // SRAM rail floor: 810 mV from the J516S runtime ADT perf-states-sram table. // Tolerate a DT without the property on G15 only. - prop!("apple,min-sram-microvolt", t6030::MIN_SRAM_MICROVOLT) + prop!("apple,min-sram-microvolt", G15_MIN_SRAM_MICROVOLT) } else { prop!("apple,min-sram-microvolt") }, diff --git a/drivers/gpu/drm/asahi/m3/hw/agx3.rs b/drivers/gpu/drm/asahi/m3/hw/agx3.rs index cb0c4fdbfa012..f5e6ad607ec25 100644 --- a/drivers/gpu/drm/asahi/m3/hw/agx3.rs +++ b/drivers/gpu/drm/asahi/m3/hw/agx3.rs @@ -2,7 +2,7 @@ #[cfg(not(test))] -use crate::identity::{GpuGen, GpuVariant}; +use crate::m3::identity::{GpuGen, GpuVariant}; #[cfg(test)] #[path = "../identity.rs"] mod identity; diff --git a/drivers/gpu/drm/asahi/m3/initdata.rs b/drivers/gpu/drm/asahi/m3/initdata.rs index dfeffd900eb12..2348d075a731c 100644 --- a/drivers/gpu/drm/asahi/m3/initdata.rs +++ b/drivers/gpu/drm/asahi/m3/initdata.rs @@ -12,21 +12,16 @@ //! derived from observing the InitData structures that macOS generates. use crate::f32; -use crate::fw::initdata::*; +use crate::m3::g15_raw::*; use crate::fw::types::*; -use crate::module_parameters; use crate::{ - driver::AsahiDevice, - gem, - gpu, - hw, + m3::hw, mmu, // }; use kernel::error::{ Error, Result, // }; -use kernel::macros::versions; use kernel::prelude::*; use kernel::try_init; @@ -35,6 +30,10 @@ use ::pin_init::Init; /// Builder helper for the global GPU InitData. +/// The "unknown page" of the 39-bit user layout (7.1.12 `mmu::IOVA_UNK_PAGE`): two pages below +/// the top of the user range. +const IOVA_UNK_PAGE: u64 = (1 << 39) - 2 * 0x4000; + /// G15 idle-off delay (ms, Globals +0x9a0/+0x9a4) used as "never idle off" while the rings live /// in Fender scratch SRAM without a backup/restore path. /// TODO: the firmware's encoding of "never" is unknown. i32::MAX stays positive if the @@ -104,24 +103,6 @@ pub(crate) struct G15RuntimeHwDataB { } impl G15Options { - /// The options of the G15 manager backend: its validated defaults, the performance-state - /// cap (`asahi.m3_max_pstate`) and the `asahi.g15_debug` A/B switches. - fn manager(cfg: &'static hw::HwConfig) -> G15Options { - let g15 = cfg.gpu_gen == hw::GpuGen::G15; - G15Options { - layout: G15Layout::Manager, - split_pstates: g15 - && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerSplitPstates), - cap: g15.then(|| { - crate::m3_params::max_pstate_param().unwrap_or(crate::m3_params::MAX_PSTATE_DEFAULT) - }), - timestamp_base: None, - reference_ppm: g15 - && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerReferencePpm), - runtime_hwdata_b: None, - per_cluster_voltages: false, - } - } /// The published performance-state tables for `pwr`. pub(crate) fn tables(&self, pwr: &hw::PwrConfig) -> Result { @@ -318,43 +299,12 @@ impl PStateTables { } } -/// The highest GPU frequency the G15 manager backend lets the firmware use, in kHz (its -/// performance cap), or None when the GPU is not G15 or the cap cannot be applied. -pub(crate) fn g15_manager_max_frequency_khz( - cfg: &'static hw::HwConfig, - pwr: &hw::PwrConfig, -) -> Option { - if cfg.gpu_gen != hw::GpuGen::G15 { - return None; - } - let tables = G15Options::manager(cfg).tables(pwr).ok()?; - Some(tables.max_frequency_khz(pwr)) -} -pub(crate) struct InitDataBuilderG15V14_8_3<'a> { - dev: &'a AsahiDevice, - alloc: &'a mut gpu::KernelAllocators, - cfg: &'static hw::HwConfig, - dyncfg: &'a hw::DynConfig, - g15: G15Options, -} +/// The G15 InitData builders. The M3 runtime places the firmware objects itself, so only the +/// builders into plain memory are kept ([`Self::g15_contents`]). +pub(crate) struct InitDataBuilderG15V14_8_3<'a>(core::marker::PhantomData<&'a ()>); impl<'a> InitDataBuilderG15V14_8_3<'a> { - /// Create a new InitData builder - pub(crate) fn new( - dev: &'a AsahiDevice, - alloc: &'a mut gpu::KernelAllocators, - cfg: &'static hw::HwConfig, - dyncfg: &'a hw::DynConfig, - ) -> InitDataBuilderG15V14_8_3<'a> { - InitDataBuilderG15V14_8_3 { - dev, - alloc, - cfg, - dyncfg, - g15: G15Options::manager(cfg), - } - } /// Create an unknown T81xx-specific data structure. fn t81xx_data( @@ -591,47 +541,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { })) } - /// Create the HwDataA structure. This mostly contains power-related configuration. - fn hwdata_a(&mut self) -> Result> { - let init = Self::hwdata_a_init(self.cfg, self.dyncfg, &self.g15)?; - let dev = self.dev; - let dyncfg = self.dyncfg; - self.alloc - .private - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { - init.chain(move |raw| { - if !dyncfg.hw_data_b.is_empty() { - unsafe { - let mut matches: bool = true; - let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataAG15V14_8_3 as *const u8, - core::mem::size_of::(), - ); - if sla.len() != dyncfg.hw_data_a.len() { - matches = false; - dev_err!( - dev.as_ref(), - "!!! Hwdata A size mismatch: {} {}", - sla.len(), - dyncfg.hw_data_a.len(), - ); - } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { - if sla[i] != dyncfg.hw_data_a[i] { - matches = false; - dev_err!(dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); - break; - } - } - if matches { - dev_info!(dev.as_ref(), "!!! Hwdata A match"); - } - } - } - Ok(()) - }) - }) - } /// The runtime backend's HwDataA words that the manager layout leaves at their defaults: the /// shader-engine controller block at +0x10b4 in its G15 order (all inputs from the device @@ -698,30 +607,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { raw.unk_429c[0] = 1; } - // Compare a structure against the copy m1n1 left in the reserved-memory blob, if any - // (debug aid, same as the G13/G14 builders do inline). - fn compare_blob(dev: &AsahiDevice, what: &str, raw: &T, blob: &[u8]) { - if blob.is_empty() { - return; - } - // SAFETY: `raw` is a fully initialized firmware structure; viewing it as bytes is fine. - let sla = unsafe { - core::slice::from_raw_parts(raw as *const T as *const u8, core::mem::size_of::()) - }; - if sla.len() != blob.len() { - dev_err!( - dev.as_ref(), - "!!! {} size mismatch: {} {}", - what, - sla.len(), - blob.len() - ); - } - match sla.iter().zip(blob.iter()).position(|(a, b)| a != b) { - Some(i) => dev_err!(dev.as_ref(), "!!! {} first mismatch: {i}", what), - None => dev_info!(dev.as_ref(), "!!! {} match", what), - } - } // Build the G15 (14.8.3) HwDataB contents without allocating the object. Board values whose // source is not known yet are left 0 and marked TODO. @@ -737,7 +622,8 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { let runtime_hwdata_b = g15.runtime_hwdata_b; let timestamp_base = match g15.timestamp_base { Some(base) => base, - None => mmu::kern_iova(cfg, gpu::IOVA_KERN_TIMESTAMP_RANGE.start), + // The runtime always names its timestamp arena. + None => return Err(EINVAL), }; Ok(try_init!(raw::HwDataBG15V14_8_3 { // VA-layout words, +0x0/+0x8 @@ -750,7 +636,7 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { // "Unknown page": the G15 firmware layout uses 0x2ff_ffff8000 (+0x20) in its // 42-bit user layout; the manager keeps a 39-bit user range and maps the page // at IOVA_UNK_PAGE, so it passes that. - unk_30: U64(mmu::IOVA_UNK_PAGE), + unk_30: U64(IOVA_UNK_PAGE), // FW VA of the timestamp area, +0x28 timestamp_area_base: U64(timestamp_base), // TODO: CSC matrices (+0x38) and border-colour pointer (+0x638) @@ -960,20 +846,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { raw.unk_1818 = words.unk_1818; } - // Create the G15 (14.8.3) HwDataB structure. - fn hwdata_b(&mut self) -> Result> { - let init = Self::hwdata_b_init(self.cfg, self.dyncfg, &self.g15)?; - let dev = self.dev; - let dyncfg = self.dyncfg; - self.alloc - .private - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { - init.chain(move |raw| { - Self::compare_blob(dev, "Hwdata B", &*raw, &dyncfg.hw_data_b); - Ok(()) - }) - }) - } // Build the G15 (14.8.3) Globals contents without allocating the object. Only the fields // with a known init value are set; the rest start at 0. @@ -1102,123 +974,7 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { Ok(()) } - // Create the G15 (14.8.3) Globals structure. - fn globals(&mut self) -> Result> { - let init = Self::globals_init(self.cfg, self.dyncfg, &self.g15)?; - let dev = self.dev; - let dyncfg = self.dyncfg; - self.alloc - .private - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { - init.chain(move |raw| { - Self::compare_blob(dev, "Globals", &*raw, &dyncfg.hw_globals); - Ok(()) - }) - }) - } - // Create the G15 (14.8.3) RuntimePointers structure. The ring descriptors are filled in by - // the GPU manager. - fn runtime_pointers(&mut self) -> Result> { - let hwa = self.hwdata_a()?; - let hwb = self.hwdata_b()?; - let bufmgr_fw_addr = mmu::kern_iova(self.cfg, gpu::IOVA_KERN_GPU_BUFMGR_HIGH); - let rp_2d0 = self.g15.runtime_pointers_2d0(); - - let mut buffer_mgr_ctl = gem::new_kernel_object(self.dev, 0x4000)?; - buffer_mgr_ctl.vmap()?.memset(0); - - GpuObject::new_init_prealloc( - self.alloc.private.alloc_object()?, - |_ptr| { - let alloc = &mut *self.alloc; - try_init!(RuntimePointersG15V14_8_3 { - // G15 E1/E2/E3 are 0xc10/0x1248/0xe10 bytes; the G13 - // GpuGlobalStatsVtx/GpuStatsComp buffers are larger, which is harmless. - stats <- { - let alloc = &mut *alloc; - try_init!(StatsG15V14_8_3 { - vtx: alloc.private.new_default::()?, - frag: alloc.private.new_init( - pin_init::init_zeroed::(), - |_inner, _ptr| { - try_init!(raw::GpuGlobalStatsFragG15V14_8_3 { - total_cmds: 0, - unk_4: 0, - stats: Default::default(), - }) - } - )?, - comp: alloc.private.new_default::()?, - }) - }, - - hwdata_a: hwa, - hwdata_b: hwb, - - // BRN (erratum) table. - // TODO: size and contents unknown; one zeroed page (no errata). - brn_table: alloc.private.array_empty_tagged(0x4000, b"IBRN")?, - // E7: 0x88 bytes, zeroed - unkptr_e7: alloc.private.array_empty_tagged(0x88, b"IE7_")?, - // E5: 0x60 bytes, zeroed - unkptr_e5: alloc.private.array_empty_tagged(0x60, b"IE5_")?, - // UMA page-pool descriptor table (RuntimePointers +0x2c0/+0x2c8). - // TODO: size/format unknown; one zeroed page. The firmware takes a - // GPU VA and a FW VA; both point at this kernel mapping for now. - uma_pool_desc: alloc.shared.array_empty_tagged(0x4000, b"IUMA")?, - - buffer_mgr_ctl, - buffer_mgr_ctl_low_mapping: None, - buffer_mgr_ctl_high_mapping: None, - }) - }, - |inner, _ptr| { - try_init!(raw::RuntimePointersG15V14_8_3 { - hwdata_b: inner.hwdata_b.gpu_pointer(), - brn_table: inner.brn_table.gpu_pointer(), - unkptr_e7: inner.unkptr_e7.gpu_pointer(), - - pipes: Default::default(), - device_control: Default::default(), - event: Default::default(), - fw_log: Default::default(), - ktrace: Default::default(), - stats: Default::default(), - fwlog_buf: None, - __pad_200: Default::default(), - // FWLog enable: the firmware writes no log records while this is 0. - // Opt-in via asahi.g15_debug bit 43. - unk_230: crate::m3_params::g15_debug(crate::m3_params::G15Debug::FwLog) as u32, - - stats_vtx: inner.stats.vtx.gpu_pointer(), - stats_frag: inner.stats.frag.gpu_pointer(), - stats_comp: inner.stats.comp.gpu_pointer(), - unkptr_e5: inner.unkptr_e5.gpu_pointer(), - __pad_254: Default::default(), - // Optional pointer, left 0 - unkptr_2a8: U64(0), - buffer_mgr_ctl_gpu_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_LOW), - buffer_mgr_ctl_fw_addr: U64(bufmgr_fw_addr), - uma_pool_desc_gpu_addr: U64(inner.uma_pool_desc.gpu_va().get()), - uma_pool_desc_fw_addr: U64(inner.uma_pool_desc.gpu_va().get()), - // TODO: board configuration words; +0x2d4 left 0. - unk_2d0: rp_2d0, - unk_2d4: 0, - __pad_2d8: Default::default(), - // +0x3b0: 0xff, then 0x90 zero bytes - cswitch_hist_ctl: 0xff, - unk_3b1: Default::default(), - hwdata_a: inner.hwdata_a.gpu_pointer(), - // TODO: 16 board configuration bytes; left 0. - unk_449: Default::default(), - __pad_459: Default::default(), - unk_46d: Default::default(), - __pad_48d: Default::default(), - }) - }, - ) - } // Fill the G15 P0 status block. Host-initialised words are 0 except +0x45c4, which arms the // firmware work-scheduler callbacks, and in the runtime layout the noise-suppression idle @@ -1230,13 +986,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { } } - // Create the G15 P0 status block. - fn status_block(&mut self) -> Result> { - let g15 = self.g15; - let mut block = self.alloc.shared.new_default::()?; - block.with_mut(|raw, _inner| Self::status_block_fill(raw, &g15)); - Ok(block) - } // Build the G15 P1 power-controller block without allocating it. The fields keep the order // and the formulas of the G13 13.5 Globals power fields (inferred). @@ -1306,18 +1055,7 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { }) } - // Create the G15 P1 power-controller block. - fn power_ctl_block(&mut self) -> Result> { - let init = Self::power_ctl_block_init(self.cfg, self.dyncfg, &self.g15); - self.alloc - .shared - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| init) - } - // Create the G15 P2 debug block. ktrace stays disabled, as on G13. - fn debug_block(&mut self) -> Result> { - self.alloc.shared.new_default::() - } /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. fn uat_level_info( @@ -1349,66 +1087,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { ] } - /// Build the top-level InitData object. - #[inline(never)] - pub(crate) fn build(&mut self) -> Result>> { - { - let pwr = &self.dyncfg.pwr; - let tables = self.g15.tables(pwr)?; - dev_info!( - self.dev.as_ref(), - "G15: performance cap {} -> {} state {} of 1..={} ({} MHz)\n", - self.g15.cap.unwrap_or(0), - if tables.secondary.is_some() { "voltage-sorted" } else { "device-tree" }, - tables.max, - tables.len - 1, - tables.max_frequency_khz(pwr) / 1000 - ); - } - let runtime_pointers = self.runtime_pointers()?; - let globals = self.globals()?; - // 14.x: FwStatus lives inside the P0 status block - let fw_status = self.status_block()?; - let power_ctl_block = self.power_ctl_block()?; - let debug_block = self.debug_block()?; - let shared_ro = &mut self.alloc.shared_ro; - - let obj = self.alloc.private.new_init( - try_init!(InitDataG15V14_8_3 { - unk_buf: shared_ro.array_empty_tagged(0x4000, b"IDTA")?, - runtime_pointers, - globals, - fw_status, - power_ctl_block, - debug_block, - }), - |inner, _ptr| { - let cfg = &self.cfg; - try_init!(raw::InitDataG15V14_8_3 { - ver_info: Array::new([0x0490, 0x8380, 0xe21e, 0x0c08]), - unk_buf: inner.unk_buf.gpu_pointer(), - unk_8: 0, - unk_c: 0, - runtime_pointers: inner.runtime_pointers.gpu_pointer(), - globals: inner.globals.gpu_pointer(), - // InitData +0x28/+0x2c: u32 0, u32 1 - unk_28: 0, - unk_2c: 1, - uat_page_size: 0x4000, - uat_page_bits: 14, - uat_num_levels: 3, - uat_level_info: Array::new(Self::uat_levels(cfg)), - // TODO: pmap descriptor +0x7c, meaning unknown; left 0. - unk_94: 0, - __pad_98: Default::default(), - debug_block: inner.debug_block.gpu_pointer(), - status_block: inner.fw_status.gpu_pointer(), - power_ctl_block: inner.power_ctl_block.gpu_pointer(), - }) - }, - )?; - Ok(KBox::new(obj, GFP_KERNEL)?) - } } /// G15 InitData contents built into plain memory, for a backend that places the firmware objects diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs index 2fa384c910688..1893c113cf2a7 100644 --- a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs @@ -13,18 +13,18 @@ use kernel::{ use crate::{ float::F32, - fw::initdata::raw, - hw, initdata, - initdata::{G15Layout, G15Options}, - m3_config::Records, - m3_firmware::Firmware, - m3_init_layout as init, - m3_init_storage as storage, - m3_memory::Buffer, - m3_params::{self, G15Debug, InitDataParam, ThermalMode}, - m3_soc::{IoMapping, PstateTable, Soc}, - t8122_hwdata, - t8122_start::Experiment, + m3::g15_raw::raw, + m3::hw, m3::initdata, + m3::initdata::{G15Layout, G15Options}, + m3::m3_config::Records, + m3::m3_firmware::Firmware, + m3::m3_init_layout as init, + m3::m3_init_storage as storage, + m3::m3_memory::Buffer, + m3::m3_params::{self, G15Debug, InitDataParam, ThermalMode}, + m3::m3_soc::{IoMapping, PstateTable, Soc}, + m3::t8122_hwdata, + m3::t8122_start::Experiment, }; // The T8122 HwDataA +0x418c block (`t8122_hwdata`) covers exactly the G13-era leakage @@ -275,7 +275,7 @@ pub(crate) struct Contents { /// map read-write) except in the T8122 start experiment. pub(crate) read_only_slots: u32, /// The HwData object's placement and mapping (`Soc::hwdata_object`). - pub(crate) hwdata_object: Option, + pub(crate) hwdata_object: Option, } fn read_u32(bytes: &[u8], offset: usize) -> Result { @@ -296,36 +296,27 @@ impl Contents { /// any GPU register is touched. /// /// `experiment` is the T8122 start experiment's values (`t8122_start`), on an armed T8122 - /// only; it then stands in for the SoC table's missing ones. `t6031` is the same for an - /// armed T6031 (`t6031_start`). + /// only; it then stands in for the SoC table's missing ones. pub(crate) fn select( - pdev: &platform::Device, + pdev: &platform::Device>, firmware: &Firmware, soc: &Soc, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, ) -> Result { let dev = pdev.as_ref(); let param = m3_params::initdata_param(); - check_version(dev, firmware, experiment, t6031)?; + check_version(dev, firmware, experiment)?; let iomaps = match experiment { Some(e) => e.iomaps(), - None => match t6031 { - Some(e) => e.iomaps(), - None => soc.iomaps.ok_or(ENODEV)?, - }, + None => soc.iomaps.ok_or(ENODEV)?, }; storage::validate_iomaps(iomaps).map_err(|_| EINVAL)?; - let images = build_images(dev, firmware, soc, experiment, t6031, iomaps)?; + let images = build_images(dev, firmware, soc, experiment, iomaps)?; let hwdata = images.get(HWDATA).and_then(|i| i.as_deref()).ok_or(EINVAL)?; - let pstates = PstatePolicy::new( - dev, - hwdata, - experiment.map(|e| e.pstate_cap()).or_else(|| t6031.map(|e| e.pstate_cap())), - )?; + let pstates = PstatePolicy::new(dev, hwdata, experiment.map(|e| e.pstate_cap()))?; let table = match soc.pstates { PstateTable::Fixed { states, top_mhz } => (states, top_mhz), - PstateTable::DeviceTree => crate::m3_board::opp_table_shape(pdev, soc).ok_or_else(|| { + PstateTable::DeviceTree => crate::m3::m3_board::opp_table_shape(pdev, soc).ok_or_else(|| { dev_err!( dev, "M3: the device tree's operating points give no performance-state table\n" @@ -346,7 +337,7 @@ impl Contents { return Err(EINVAL); } // The published states never go above the device tree's highest operating point. - if let Some(dt_khz) = crate::m3_board::max_frequency_khz(pdev) { + if let Some(dt_khz) = crate::m3::m3_board::max_frequency_khz(pdev) { if 1000 * pstates.max_mhz > dt_khz { dev_err!( dev, @@ -377,14 +368,8 @@ impl Contents { pstates.max, pstates.max_mhz ); - let read_only_slots = experiment - .map(|e| e.read_only_slots()) - .or_else(|| t6031.map(|e| e.read_only_slots())) - .unwrap_or(0); - let hwdata_object = match t6031 { - Some(e) => e.hwdata_object(), - None => soc.hwdata_object, - }; + let read_only_slots = experiment.map(|e| e.read_only_slots()).unwrap_or(0); + let hwdata_object = soc.hwdata_object; Ok(Contents { images, pstates, iomaps, table, read_only_slots, hwdata_object }) } @@ -435,7 +420,6 @@ fn check_version( dev: &device::Device, firmware: &Firmware, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, ) -> Result { let version = constructed_version()?; if version != firmware.initdata_magic && experiment.is_some() { @@ -447,15 +431,6 @@ fn check_version( ); return Ok(()); } - if version != firmware.initdata_magic && t6031.is_some() { - dev_warn!( - dev, - "M3 G15C start: InitData version {:#x} replaces the constructed {:#x}\n", - firmware.initdata_magic, - version - ); - return Ok(()); - } if version != firmware.initdata_magic { dev_err!( dev, @@ -572,7 +547,6 @@ fn fill_io_mappings( dev: &device::Device, cfg: &'static hw::HwConfig, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, mappings: &[IoMapping], iomaps: &[storage::IoMap], hwdata: &mut [u8], @@ -585,10 +559,9 @@ fn fill_io_mappings( let virt = entry + offset_of!(raw::IOMapping, virt_addr); // The same register block as the manager's table. let same_block = cfg.io_mappings.get(slot).and_then(|m| m.as_ref()).is_some_and(|m| { - let base = match (experiment, t6031) { - (Some(e), _) => e.io_block(slot, m.base as u64), - (None, Some(e)) => e.io_block(slot, m.base as u64), - (None, None) => m.base as u64, + let base = match experiment { + Some(e) => e.io_block(slot, m.base as u64), + None => m.base as u64, }; base & !0x3fff == phys & !0x3fff && m.writable == writable }); @@ -720,7 +693,7 @@ fn whole(v: F32) -> u32 { /// The boot loader's decoded GPU leakage fuse values (`Soc::leak_fuse`), logged when it exported /// them. Returns them when the boot loader's switch is set. With the switch set and no two /// valid values, refuses: the boot loader then used neither the fuse nor the stand-in. -fn leak_fuse(dev: &device::Device, lf: &crate::m3_soc::LeakFuse) -> Result> { +fn leak_fuse(dev: &device::Device, lf: &crate::m3::m3_soc::LeakFuse) -> Result> { let chosen = kernel::of::chosen(); let switch = chosen .as_ref() @@ -790,29 +763,19 @@ fn build_images( firmware: &Firmware, soc: &Soc, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, iomaps: &[storage::IoMap], ) -> Result>>> { check_layout().inspect_err(|_| { dev_err!(dev, "M3: the constructed InitData records do not match the G15 InitData structures\n") })?; - let cfg: &'static hw::HwConfig = match t6031 { - Some(e) => e.hwcfg(), - None => soc.hwcfg.ok_or(ENODEV)?, - }; + let cfg: &'static hw::HwConfig = soc.hwcfg.ok_or(ENODEV)?; let io_mappings = match experiment { Some(e) => e.io_mappings(), - None => match t6031 { - Some(e) => e.io_mappings(), - None => soc.io_mappings.ok_or(ENODEV)?, - }, + None => soc.io_mappings.ok_or(ENODEV)?, }; let runtime_hwdata_b = match experiment { Some(e) => e.hwdata_b(), - None => match t6031 { - Some(e) => e.hwdata_b(), - None => soc.hwdata_b.ok_or(ENODEV)?, - }, + None => soc.hwdata_b.ok_or(ENODEV)?, }; let mut pwr = hw::PwrConfig::load(dev, cfg).inspect_err(|e| { dev_err!(dev, "M3: cannot read the GPU power configuration from the device tree ({:?})\n", e) @@ -841,10 +804,7 @@ fn build_images( } // A SoC whose power model is still a stand-in caps the power target: every operating point's // power is scaled by the same factor, so the relative powers stay the boot loader's. - let power_cap_mw = match t6031 { - Some(e) => Some(e.power_cap_mw()), - None => soc.power_target_cap_mw, - }; + let power_cap_mw = soc.power_target_cap_mw; if let Some(cap) = power_cap_mw { let max = pwr.max_power_mw; if max > cap { @@ -863,7 +823,7 @@ fn build_images( let node = dev.of_node().ok_or(ENODEV)?; let dyncfg = hw::DynConfig { uat_ttb_base: firmware.resources.regions[0].base, - id: soc_identity(soc, cfg, t6031.and_then(|e| e.revision_id())), + id: soc_identity(soc, cfg, None), pwr, firmware_version: node .get_property::>(c_str!("apple,firmware-version")) @@ -877,7 +837,7 @@ fn build_images( split_pstates: true, // The cap is applied to the uploaded objects, for either source (`publish_pstates`). cap: None, - timestamp_base: Some(crate::agx_memory::TIMESTAMP_RANGE.start), + timestamp_base: Some(crate::m3::agx_memory::TIMESTAMP_RANGE.start), reference_ppm: true, runtime_hwdata_b: Some(runtime_hwdata_b), per_cluster_voltages: soc.per_cluster_voltages, @@ -922,12 +882,6 @@ fn build_images( write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; dev_info!(dev, "M3 G15G start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); - } else if let Some(e) = t6031 { - type B = raw::HwDataBG15V14_8_3; - let (a, b) = e.unit_masks(); - write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; - write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; - dev_info!(dev, "M3 G15C start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); } write(&mut hwdata, HWDATA_A, initdata::raw_bytes(&*c.hwdata_a))?; // The system counter value at InitData creation, the base of the firmware's first power and @@ -953,14 +907,6 @@ fn build_images( m.fast_die, m.alarm ); - } else if let Some(e) = t6031 { - let (fast, alarm) = e.mtr_masks(dev)?; - type A = raw::HwDataAG15V14_8_3; - for at in [offset_of!(A, fast_die0_sensor_mask), offset_of!(A, fast_die0_sensor_mask_2)] { - write(&mut hwdata, HWDATA_A + at, &fast.to_le_bytes())?; - } - write(&mut hwdata, HWDATA_A + offset_of!(A, fast_die0_sensor_mask_alt), &alarm.to_le_bytes())?; - dev_info!(dev, "M3: MTR sensor masks: fast-die {:#x}, alarm {:#x}\n", fast, alarm); } // The second leakage fuse value: the first entry of HwDataA's third leakage table (the first // table holds the core coefficient, from the builder). @@ -969,23 +915,15 @@ fn build_images( let at = HWDATA_A + offset_of!(A, cluster_tables) + 2 * size_of::<[u32; 8]>(); write(&mut hwdata, at, &second.to_bits().to_le_bytes())?; } - // The SoC's own values for words the shared builder writes otherwise. An armed T6031 copies - // the T8122 words only when asahi.t6031_fw_words=t8122. - let (hwdata_words, globals_words): (&[(usize, u32)], &[(usize, u32)]) = match t6031 { - Some(e) if e.fw_words() => ( - &crate::m3_soc::T8122_HWDATA_WORDS, - &crate::m3_soc::T8122_GLOBALS_WORDS, - ), - Some(_) => (&[], &[]), - None => (soc.hwdata_words, soc.globals_words), - }; + // The SoC's own values for words the shared builder writes otherwise. + let (hwdata_words, globals_words) = (soc.hwdata_words, soc.globals_words); for &(at, value) in hwdata_words { write(&mut hwdata, at, &value.to_le_bytes())?; } // T8122 with the G15 14.8.3 firmware (the only image the runtime accepts there): the 20 - // words of crate::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. - if crate::t8122_start::is_t8122(soc) - && core::ptr::eq(firmware.layout, &crate::m3_firmware::T8122_LAYOUT) + // words of crate::m3::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. + if crate::m3::t8122_start::is_t8122(soc) + && core::ptr::eq(firmware.layout, &crate::m3::m3_firmware::T8122_LAYOUT) && cfg.gpu_gen == hw::GpuGen::G15 { if m3_params::t8122_hwdata_418c() { @@ -1006,7 +944,7 @@ fn build_images( } // On T8122, log and require the two HwDataB words the firmware's power management depends // on: it powers the GPU cores up for a job only when +0xa38 and +0xa40 are both nonzero. - if core::ptr::eq(soc, &crate::m3_soc::T8122) { + if core::ptr::eq(soc, &crate::m3::m3_soc::T8122) { type B = raw::HwDataBG15V14_8_3; let a38 = read_u32(&hwdata, offset_of!(B, unk_460))?; let a40 = read_u32(&hwdata, offset_of!(B, unk_468))?; @@ -1019,7 +957,7 @@ fn build_images( return Err(EINVAL); } } - fill_io_mappings(dev, cfg, experiment, t6031, io_mappings, iomaps, &mut hwdata)?; + fill_io_mappings(dev, cfg, experiment, io_mappings, iomaps, &mut hwdata)?; images[HWDATA] = Some(hwdata); let mut globals = zeroed(GLOBALS)?; @@ -1047,7 +985,7 @@ fn build_images( if given { " (asahi.m3_ut_engagement)" } else { "" } ); } - if crate::m3_globals_debug::apply(soc.chip_id,m3_params::globals_debug_off(),&mut globals) + if crate::m3::m3_globals_debug::apply(soc.chip_id,m3_params::globals_debug_off(),&mut globals) .map_err(|_|EINVAL)? { dev_info!(dev,"M3: T6030 Globals debug word cleared at +0x2c\n"); } @@ -1237,7 +1175,7 @@ pub(crate) fn publish_runtime_cap(globals: &mut Buffer, cap: u32) -> Result { for off in GLOBALS_RUNTIME_CAP { globals.u32(off, 100 * cap)?; } - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); Ok(()) } diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs index 5ecfd820b4e4f..c44b2a42d99d5 100644 --- a/drivers/gpu/drm/asahi/m3/m3_board.rs +++ b/drivers/gpu/drm/asahi/m3/m3_board.rs @@ -1,16 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Board facts for the M3 runtime, read from the hardware and the device tree instead of being -//! fixed to one board. The per-SoC facts are in [`crate::m3_soc`]. +//! fixed to one board. The per-SoC facts are in [`crate::m3::m3_soc`]. use kernel::{bindings, c_str, device, device::Core, io::resource::Resource, of, platform, prelude::*, uapi}; -use crate::m3_resources::{ +use crate::m3::m3_resources::{ reserved_resource, Region, Resources, // }; -use crate::m3_soc::Soc; +use crate::m3::m3_soc::Soc; /// Whether the root node's compatible list contains `compatible`. fn board_is(compatible: &[u8]) -> bool { @@ -60,7 +60,7 @@ pub(crate) fn core_masks(soc: &Soc, mask: u64) -> [u32; uapi::DRM_ASAHI_MAX_CLUS /// The highest GPU frequency in the device tree OPP table, in kHz, or None when the GPU node /// has no populated OPP table. -pub(crate) fn max_frequency_khz(pdev: &platform::Device) -> Option { +pub(crate) fn max_frequency_khz(pdev: &platform::Device>) -> Option { let node = pdev.as_ref().of_node()?; let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; let mut max_hz: u64 = 0; @@ -77,7 +77,7 @@ pub(crate) fn max_frequency_khz(pdev: &platform::Device) -> Option { /// Every cluster of a point must have the same voltage, except on a SoC with per-cluster /// voltages, where the highest `opp-microvolt` cell orders the point. None without a readable /// OPP table. -pub(crate) fn opp_table_shape(pdev: &platform::Device, soc: &Soc) -> Option<(u32, u32)> { +pub(crate) fn opp_table_shape(pdev: &platform::Device>, soc: &Soc) -> Option<(u32, u32)> { let node = pdev.as_ref().of_node()?; let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; let mut points = KVec::new(); @@ -99,7 +99,7 @@ pub(crate) fn opp_table_shape(pdev: &platform::Device, soc: &Soc) -> Optio }; points.push((hz, uv), GFP_KERNEL).ok()?; } - crate::t8122_admission::opp_table_shape(points) + crate::m3::t8122_admission::opp_table_shape(points) } /// Register windows the runtime maps on `soc`: name, base, minimum size. The runtime uses the @@ -340,7 +340,7 @@ pub(crate) fn static_region(node: &of::Node, name: &CStr) -> Option, soc: &Soc) -> Result { +pub(crate) fn admit(pdev: &platform::Device>, soc: &Soc) -> Result { let dev = pdev.as_ref(); let refuse = |what: &str, err: Error| { dev_info!(dev, "M3: not admitted: {}\n", what); @@ -460,14 +460,14 @@ pub(crate) fn admit(pdev: &platform::Device, soc: &Soc) -> Result) -> bool { +pub(crate) fn has_pmp_link(pdev: &platform::Device>) -> bool { pdev.as_ref() .of_node() .is_some_and(|node| node.get_property::(c_str!("apple,pmp")).is_ok()) } -pub(crate) use crate::m3_firmware::{KnownImage, KNOWN_IMAGES, KNOWN_IMAGES_T8122}; -use crate::m3_firmware::image_info; +pub(crate) use crate::m3::m3_firmware::{KnownImage, KNOWN_IMAGES_T8122}; +use crate::m3::m3_firmware::image_info; fn sha256(data: &[u8]) -> [u8; 32] { let mut digest = [0u8; 32]; @@ -551,9 +551,8 @@ pub(crate) fn region_is_nomap(node: &of::Node, name: &CStr) -> bool { /// board, a render node there would let a stock userspace driver submit work the board may not /// run correctly. `auto` (-1) registers it only on the boards the runtime was validated on. pub(crate) fn expose_render_node(soc: &Soc) -> bool { - match *crate::module_parameters::m3_expose.value() { - 0 => false, - v if v > 0 => true, - _ => runtime_validated_board(soc), + match crate::m3::m3_params::expose() { + Some(expose) => expose, + None => runtime_validated_board(soc), } } diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs index 6661222530a38..8fc7791e939ba 100644 --- a/drivers/gpu/drm/asahi/m3/m3_client.rs +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -13,7 +13,6 @@ use core::sync::atomic::{ AtomicU32, - AtomicU64, Ordering, // }; @@ -23,8 +22,7 @@ use kernel::{ }; use crate::{ - driver::AsahiDevice, - agx_uapi::{ + m3::agx_uapi::{ GpuAccess, GpuAddressSpace, // }, @@ -38,28 +36,14 @@ const AGX3_PARAMS_SIZE: u64 = core::mem::size_of:: Self { - Self { - params_size: AtomicU64::new(0), - } - } - - /// Record a successful GET_PARAMS for the global parameters. - pub(crate) fn note_params(&self, size: u64) { - self.params_size.fetch_max(size, Ordering::Relaxed); - } - - /// Whether this file may create a VM. Only G15 devices are gated. - pub(crate) fn admit_vm(&self, device: &AsahiDevice, flags: u32) -> Result { - let size = self.params_size.load(Ordering::Relaxed); - if crate::g16_profile::client_admitted(device.gpu()?.params()?.gpu_hal_generation, flags, size >= AGX3_PARAMS_SIZE) { +/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for the +/// whole global parameter structure (so it knows this GPU runs AGX3 command streams), and pass +/// VM_CREATE flags 0 (legacy HAL). `size` is the largest GET_PARAMS size the file asked for. +pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) -> Result { + { + // The positive acknowledgement of the command ABI (7.1.12 `g16_profile::client_admitted`): + // GET_PARAMS coverage, and VM_CREATE flags 0 on the legacy-HAL M3 runtime. + if size >= AGX3_PARAMS_SIZE && flags == 0 { return Ok(()); } let refusals = REFUSALS.fetch_add(1, Ordering::Relaxed).saturating_add(1); @@ -74,7 +58,7 @@ impl ClientGate { "" }; dev_info!( - device.as_ref(), + device, "M3: refusing a VM for {}[{}]: it asked GET_PARAMS for {} of the {} parameter bytes, so it cannot know this GPU runs AGX3 command streams; M3-aware userspace is needed{}\n", name, task.group_leader().pid(), @@ -200,7 +184,9 @@ pub(crate) fn check_compute_launches( /// `Vm::read_mapped_bytes` holds the VM execution lock from translation until the copy, so the /// pages cannot be unmapped and freed while they are cleaned and read. fn read_through_cache(vm: &mmu::Vm, iova: u64, out: &mut [u8]) -> Result { - vm.read_mapped_bytes(iova, out, |src, len| clean_invalidate(src as usize, len)) + // 7.2's `Vm::read_user` cleans and invalidates each line to the point of coherency before + // the copy, under the VM execution lock. + vm.read_user(iova, out) } /// Clean and invalidate the data cache lines covering `start..start + len` to the point of diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs index 08114c4d8cc8a..e06fcded9047e 100644 --- a/drivers/gpu/drm/asahi/m3/m3_completion.rs +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -3,7 +3,7 @@ //! Completion work owned only by the M3 runtime. use kernel::{prelude::*, sync::Arc, workqueue::{self, impl_has_work, new_work, Work, WorkItem}}; -use crate::{driver, m3_drm::Shared}; +use crate::m3::{m3_drm::Shared, m3_rtkit::State}; #[pin_data] pub(crate) struct Completion { @@ -23,7 +23,7 @@ impl_has_work! { impl HasWork for Completion { self.work } } impl WorkItem for Completion { type Pointer = Arc; fn run(owner: Arc) { - use crate::m3_runtime::Service; + use crate::m3::m3_runtime::Service; // Snapshots after a notification before sleeping for the next one: the completion // stamps can trail the firmware's event message. const TRAILING_POLLS: u32 = 4; @@ -59,8 +59,8 @@ impl WorkItem for Completion { } } } -pub(crate) fn queue(dev: &driver::AsahiDevice) { - if let Some(work) = dev.completion.as_ref() { +pub(crate) fn queue(state: &State) { + if let Some(work) = state.completion.as_ref() { let _ = workqueue::system_highpri().enqueue(work.clone()); } } diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index e7469d31b496f..eab8bf71f7bcd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -1,15 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Owned RTKit 2419 M3 compute graph, populated with a validated user CDM. use kernel::prelude::*; -use crate::m3_queue_layout as queue; -use crate::m3_state_layout as state; -use crate::{m3_compute_layout as command, m3_compute_sequence as microsequence, - m3_pool_layout::GpuRegion, m3_init_layout::Region}; -use crate::{driver,mmu,agx_memory::{self},m3_memory::Buffer,m3_compute_storage as storage}; -use crate::m3_timeline as timeline; +use crate::m3::m3_queue_layout as queue; +use crate::m3::m3_state_layout as state; +use crate::{m3::m3_compute_layout as command, m3::m3_compute_sequence as microsequence, + m3::m3_pool_layout::GpuRegion, m3::m3_init_layout::Region}; +use crate::{driver,mmu,m3::agx_memory::{self},m3::m3_memory::Buffer,m3::m3_compute_storage as storage}; +use crate::m3::m3_timeline as timeline; use storage::{NOTIFIER,PREEMPTION,GPU_CONTEXT,JOB_LIST,QUEUE_SCRATCH,TIMESTAMPS, TAIL_SCRATCH,MICROSEQUENCE_SCRATCH,Mapping}; -const UMA_POOL:crate::m3_pool::Owners=crate::m3_pool::Owners { +const UMA_POOL:crate::m3::m3_pool::Owners=crate::m3::m3_pool::Owners { backing:storage::POOL_BACKING,pages:storage::POOL_PAGES,table:storage::POOL_TABLE, manager:storage::POOL,counter:storage::COUNTER,table_index:2, }; @@ -40,11 +40,14 @@ pub(crate) struct Compute { barriers: Option, /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). registers: command::RegisterSet, + /// Last: drops after the command aliases. + _lower_tlbi: crate::m3::m3_memory::LowerTlbi, } impl Compute { - pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, vm: &mmu::Vm, - stats: Region, control: crate::agx_compute::Control, + pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, + stats: Region, control: crate::m3::agx_compute::Control, registers: command::RegisterSet) -> Result { + let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new(); let mut client_command=None; @@ -101,10 +104,10 @@ impl Compute { // the invalidation. crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; - Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers}) + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,_lower_tlbi:lower_tlbi}) } fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, - control: crate::agx_compute::Control, sequence: u64, slot: usize, + control: crate::m3::agx_compute::Control, sequence: u64, slot: usize, register_set: command::RegisterSet) -> Result { let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); @@ -164,7 +167,7 @@ impl Compute { objects[PREEMPTION].fill(0xcc) } pub(crate) fn replay(&mut self, uat: &mmu::Uat, vm: &mmu::Vm, - control: crate::agx_compute::Control) -> Result { + control: crate::m3::agx_compute::Control) -> Result { if !self.complete()? { return Err(EBUSY); } // All prior flushed stamps/events and engine idle were checked by // Runtime before entry. Keep pool/queue ownership, remap only client @@ -211,7 +214,7 @@ impl Compute { /// Append only while the previous batch is fully retired and this one /// remains unpublished. Firmware serializes each command, including its /// original completion flush, in this one compute queue. - pub(crate) fn append(&mut self,control:crate::agx_compute::Control)->Result { + pub(crate) fn append(&mut self,control:crate::m3::agx_compute::Control)->Result { if self.batch_count>=storage::SLOTS {return Err(E2BIG);} self.slot=self.batch_count; for i in storage::RESET { @@ -268,20 +271,20 @@ impl Compute { } agx_memory::publish();Ok(()) } - pub(crate) fn set_attachments(&mut self,attachments:&crate::agx_attachments::Attachments)->Result { + pub(crate) fn set_attachments(&mut self,attachments:&crate::m3::agx_attachments::Attachments)->Result { let sequence=self.offset(storage::MICROSEQUENCE)?; self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::ATTACHMENTS,&attachments.0)?; self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::HAS_ATTACHMENTS,&[u8::from(attachments.count()!=0)])?; agx_memory::publish();Ok(()) } fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } - pub(crate) fn completion_dependency(&self)->Result { - Ok(crate::m3_sync_layout::CompletionDependency { + pub(crate) fn completion_dependency(&self)->Result { + Ok(crate::m3::m3_sync_layout::CompletionDependency { stamp:queue::FirmwareVa::new(self.objects[storage::FW_STAMP].va()).map_err(|_|EINVAL)?, value:self.stamp(),event:2, }) } - pub(crate) fn reserve_barriers(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat)->Result { + pub(crate) fn reserve_barriers(&mut self,dev:&driver::AsahiDrmDevice,uat:&mmu::Uat)->Result { if self.barriers.is_none() { self.barriers=Some(Buffer::at(dev,uat.kernel_vm(),None,None,storage::SLOTS*64,false)?); } @@ -290,17 +293,17 @@ impl Compute { /// Insert a dependency before the last appended, still unpublished CDM. /// Its command ordinal and timestamp slot are unchanged; only the ring /// gains an entry. The whole batch must retire before this storage is reused. - pub(crate) fn insert_dependency(&mut self,dependency:crate::m3_sync_layout::CompletionDependency)->Result { + pub(crate) fn insert_dependency(&mut self,dependency:crate::m3::m3_sync_layout::CompletionDependency)->Result { if self.batch_count==0 || self.slot>=storage::SLOTS || dependency.event!=1 { return Err(EINVAL); } let value=self.stamp(); let barriers=self.barriers.as_mut().ok_or(EINVAL)?; let offset=self.slot*64; - let mut bytes=[0;crate::m3_sync_layout::BARRIER_SIZE]; - crate::m3_sync_layout::Barrier { + let mut bytes=[0;crate::m3::m3_sync_layout::BARRIER_SIZE]; + crate::m3::m3_sync_layout::Barrier { stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event, - self_value:value,uuid:0xc10000,dependency:crate::m3_sync_layout::Dependency::PreviousPass, + self_value:value,uuid:0xc10000,dependency:crate::m3::m3_sync_layout::Dependency::PreviousPass, }.encode(&mut bytes).map_err(|_|EINVAL)?; barriers.write(offset,&bytes)?; let address=barriers.va()+offset as u64; @@ -333,7 +336,7 @@ impl Compute { pub(crate) fn head(&self) -> u16 { self.head } pub(crate) fn ordinal(&self)->u64 {self.sequence} pub(crate) fn first(&self) -> bool { self.sequence == self.batch_count as u64 } - pub(crate) fn progress(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn progress(&mut self, dev: &driver::AsahiDrmDevice) -> Result { // Report real completions by elapsed time as well as sequence count; // batches need not land on a multiple of 128. if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { @@ -360,7 +363,7 @@ impl Compute { && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?==u32::from(self.head) && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?==u32::from(self.head)) } - pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { for off in (0..512).step_by(128) { let mut raw=[0u8;128]; self.objects[storage::COMMAND].read(off,&mut raw)?; dev_info!(dev.as_ref(),"M3 post-command {:#x} {:02x?}\n",off,raw); diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index 59277aaefb681..c4a44c4b4f3b0 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -1,12 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Owned RTKit 2419.140.12 configuration with named initialization references. use kernel::prelude::*; -use crate::m3_init_layout as init; -use crate::m3_init_storage as storage; +use crate::m3::m3_init_layout as init; +use crate::m3::m3_init_storage as storage; use storage::{REGION_A,RUNTIME_POINTERS,HARDWARE_DATA,UNKNOWN_PAIR,FWLOG_PAYLOAD, UNKNOWN_C0,UNKNOWN_C1,UNKNOWN_C3,GLOBALS,GLOBALS_POWER,CONTROL_REGION,RUNTIME_FLAGS, FW_CONTROL_STATE,FW_CONTROL_RING}; -use crate::{driver,agx_memory::{self},m3_memory::Buffer,mmu,pgtable::prot}; +use crate::{driver,m3::agx_memory::{self},m3::m3_memory::Buffer,mmu,pgtable::prot}; kernel::static_assert!(core::mem::size_of::() == init::fwlog::ENTRY_SIZE); kernel::static_assert!(core::mem::size_of::() == init::fwlog::PAYLOAD_SIZE); kernel::static_assert!(::SUB_CHANNELS == init::fwlog::CHANNELS); @@ -49,12 +49,10 @@ impl Records { pub(crate) struct Config { objects: KVec, - #[cfg(CONFIG_DEV_COREDUMP)] - fault_reserve: Option, _iomaps: KVec, pub(crate) completed_events: u64, - pstates: crate::m3_adt_config::PstateWatch, - thermal: crate::m3_thermal::Governor, + pstates: crate::m3::m3_adt_config::PstateWatch, + thermal: crate::m3::m3_thermal::Governor, request_top: u32, request_value: u32, request_checked: bool, @@ -63,47 +61,9 @@ pub(crate) struct Config { request_observed: u32, } impl Config { - /// Snapshot owned RAM only. Firmware can still update these buffers; - /// per-record time intervals explicitly describe sequential observations. - #[cfg(CONFIG_DEV_COREDUMP)] - pub(crate) fn fault_snapshot(&mut self, dev: &driver::AsahiDevice, primary: Error, gpu_pending: bool) -> Result { - let mut dump = self.fault_reserve.take().ok_or(ENOMEM)?; - dump.begin(0); - let mut cause = [0u8; 16]; - cause[..4].copy_from_slice(&1u32.to_le_bytes()); - cause[4..8].copy_from_slice(&primary.to_errno().to_le_bytes()); - cause[8..16].copy_from_slice(&u64::from(gpu_pending).to_le_bytes()); - dump.record("host-first-error", 0, cause.len(), |out| { - out.copy_from_slice(&cause); Ok(()) - })?; - for (name, index) in [ - ("root", storage::ROOT), ("runtime", RUNTIME_POINTERS), - ("hardware", HARDWARE_DATA), ("globals", GLOBALS), - ("control", CONTROL_REGION), ("flags", RUNTIME_FLAGS), - // The power-controller block, and the host-to-firmware device-control and - // firmware-control queues: their state words show whether the firmware read a - // queued control (read index) and what it was sent. - ("power", GLOBALS_POWER), - ("devctl-state", storage::DEVICE_CONTROL * 2), - ("devctl-ring", storage::DEVICE_CONTROL * 2 + 1), - ("fwctl-state", storage::FW_CONTROL_STATE), - ("fwctl-ring", storage::FW_CONTROL_RING), - ("event-state", storage::EVENT * 2), - ("event-ring", storage::EVENT * 2 + 1), - ("fwlog-state", storage::FIRMWARE_LOG * 2), - ("fwlog-ring", storage::FIRMWARE_LOG * 2 + 1), - ("fwlog-payload", FWLOG_PAYLOAD), - ("trace-state", storage::TRACE * 2), - ("trace-ring", storage::TRACE * 2 + 1), - ] { - let buffer = &mut self.objects[index]; - dump.record(name, buffer.va(), buffer.size(), |out| buffer.read(0, out))?; - } - dump.publish(dev.as_ref()) - } - pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, firmware: &crate::m3_firmware::Firmware, - contents: &crate::m3_adt_config::Contents) -> Result { + pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, firmware: &crate::m3::m3_firmware::Firmware, + contents: &crate::m3::m3_adt_config::Contents) -> Result { contents.check_for_upload()?; let mut objects = KVec::new(); let mut iomaps = KVec::new(); @@ -125,8 +85,8 @@ impl Config { for io in contents.iomaps { // Read-write unless the contents say otherwise (only the T8122 start experiment). let protection=if contents.read_only_slots==0 {prot::PROT_FW_MMIO_RW} - else {crate::t8122_start::mmio_prot(contents.read_only_slots,io.slot)}; - let map=uat.kernel_vm().map_io(io.address,io.physical.try_into()?,io.size,protection)?; + else {crate::m3::t8122_start::mmio_prot(contents.read_only_slots,io.slot)}; + let map=uat.kernel_vm().map_io_m3(io.address,io.physical.try_into()?,io.size,protection)?; let owned=init::Region::new(map.iova(),io.size).map_err(|_|EINVAL)?; let field=io.pointer_field().map_err(|_|EINVAL)?; objects[HARDWARE_DATA].u64(field,io.pointer(owned).map_err(|_|EINVAL)?)?; @@ -139,7 +99,7 @@ impl Config { // The root record carries the G15 14.8.3 InitData version, which is every T6030 // image's. Only the T8122 start experiment gives the firmware another one // (`asahi.t8122_initdata_version`). - if firmware.initdata_magic!=crate::m3_firmware::G15_V14_8_3_INITDATA { + if firmware.initdata_magic!=crate::m3::m3_firmware::G15_V14_8_3_INITDATA { objects[storage::ROOT].u64(0,firmware.initdata_magic)?; } // Vertex/tessellation loops need not retire a primitive at every @@ -171,26 +131,17 @@ impl Config { return Err(EINVAL); } let (hwdata, globals) = objects.split_at_mut(GLOBALS); - crate::m3_adt_config::publish_pstates(dev.as_ref(), &mut hwdata[HARDWARE_DATA], &mut globals[0], &contents.pstates)?; + crate::m3::m3_adt_config::publish_pstates(dev.as_ref().as_ref(), &mut hwdata[HARDWARE_DATA], &mut globals[0], &contents.pstates)?; // Firmware ktrace is for explicit detailed diagnostics. Normal games // still drain error/events, but need not generate trace records for // every firmware operation on the command-processing core. objects[RUNTIME_FLAGS].u32(0,u32::from(crate::debug::debug_enabled( crate::debug::DebugFlags::SubmitTiming)))?; - crate::m3_adt_config::check_upload(dev.as_ref(), firmware, contents.iomaps, &mut objects)?; + crate::m3::m3_adt_config::check_upload(dev.as_ref().as_ref(), firmware, contents.iomaps, &mut objects)?; agx_memory::publish(); - let thermal = crate::m3_thermal::Governor::new(dev.as_ref(), &contents.pstates)?; - #[cfg(CONFIG_DEV_COREDUMP)] - let fault_reserve = match crate::agx_fault::Dump::new_m3() { - Ok(dump) => Some(dump), - Err(error) => { - dev_warn!(dev.as_ref(), "M3: fault snapshot reserve unavailable: {:?}\n", error); - None - } - }; + let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?; Ok(Self { objects, _iomaps: iomaps, completed_events: 0, - #[cfg(CONFIG_DEV_COREDUMP)] fault_reserve, - pstates: crate::m3_adt_config::PstateWatch::new(contents.pstates), thermal, + pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal, request_top:contents.table.0,request_value:0,request_checked:false, request_check_at:None,request_writes:0,request_observed:0 }) } @@ -203,22 +154,22 @@ impl Config { objects[CONTROL_REGION].write(0, &records.control) } pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } - pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDevice)->Result { - let value=self.objects[GLOBALS].read_u32(crate::m3_globals_debug::OFFSET)?; + pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDrmDevice)->Result { + let value=self.objects[GLOBALS].read_u32(crate::m3::m3_globals_debug::OFFSET)?; dev_info!(dev.as_ref(),"M3_GLOBALS_DEBUG off={} value={} request_level={}\n", - u8::from(crate::m3_params::globals_debug_off()),value,crate::m3_params::consistent_perf()); + u8::from(crate::m3::m3_params::globals_debug_off()),value,crate::m3::m3_params::consistent_perf()); Ok(()) } pub(crate) fn performance_request(&mut self,chip:u32,busy:bool)->Result { - let Some(value)=crate::m3_consistent_perf::request(chip, - crate::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; + let Some(value)=crate::m3::m3_consistent_perf::request(chip, + crate::m3::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; if value==0 && self.request_value==0 {return Ok(());} let now=kernel::time::Instant::::now(); let elapsed=self.request_check_at.map(|last|(now-last).as_nanos()); - if !crate::m3_consistent_perf::refresh_due(value,self.request_value,elapsed) { + if !crate::m3::m3_consistent_perf::refresh_due(value,self.request_value,elapsed) { return Ok(()); } - type P=crate::fw::initdata::raw::PowerCtlBlock; + type P=crate::m3::g15_raw::raw::PowerCtlBlock; let at=core::mem::offset_of!(P,consistent_perf_state); if !self.request_checked { let mut flag=[0]; @@ -240,14 +191,14 @@ impl Config { /// Move the runtime cap by temperature (the thermal limit), then check the performance /// state the firmware reports against the ceiling and the runtime cap. Called after boot and /// after every job, by the job thread. - pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDevice, device: &crate::m3_device::Device, what: &str) -> Result { + pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDrmDevice, device: &crate::m3::m3_device::Device, what: &str) -> Result { let now = kernel::time::Instant::::now(); - if let Some(cap) = self.thermal.update(dev.as_ref(), now) { - crate::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; + if let Some(cap) = self.thermal.update(dev.as_ref().as_ref(), now) { + crate::m3::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; } let allowed = self.thermal.allowed(now); - let (over, peak) = self.pstates.check(dev.as_ref(), &mut self.objects[HARDWARE_DATA], || device.pstate_register(), what, allowed)?; - self.thermal.note_check(dev.as_ref(), now, over, peak)?; + let (over, peak) = self.pstates.check(dev.as_ref().as_ref(), &mut self.objects[HARDWARE_DATA], || device.pstate_register(), what, allowed)?; + self.thermal.note_check(dev.as_ref().as_ref(), now, over, peak)?; self.thermal.busy(now); Ok(()) } @@ -256,14 +207,14 @@ impl Config { init::Region::new(owner.va(), owner.size()).map_err(|_| EINVAL) } - pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDevice, - device: &crate::m3_device::Device, + pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDrmDevice, + device: &crate::m3::m3_device::Device, sequence: u32) -> Result { let cap = self.thermal.allowed(kernel::time::Instant::::now()); - self.pstates.log_active(dev.as_ref(), &mut self.objects[HARDWARE_DATA], + self.pstates.log_active(dev.as_ref().as_ref(), &mut self.objects[HARDWARE_DATA], device.pstate_register()?, cap, sequence)?; - type P=crate::fw::initdata::raw::PowerCtlBlock; - type G=crate::fw::initdata::raw::GlobalsG15V14_8_3; + type P=crate::m3::g15_raw::raw::PowerCtlBlock; + type G=crate::m3::g15_raw::raw::GlobalsG15V14_8_3; let request=self.objects[GLOBALS_POWER].read_u32(core::mem::offset_of!(P,consistent_perf_state))?; let cap_word=self.objects[GLOBALS].read_u32(core::mem::offset_of!(G,perf_state_cap))?; let power_if=core::mem::offset_of!(G,power_if_target); @@ -272,13 +223,13 @@ impl Config { let mut override_active=[0]; self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut override_active)?; dev_info!(dev.as_ref(),"M3_PERF_REQUEST sample={} level={} request={} cap_word={} target1={} target2={} writes={} observed={} override_active={}\n", - sequence,crate::m3_params::consistent_perf(),request,cap_word,target1,target2, + sequence,crate::m3::m3_params::consistent_perf(),request,cap_word,target1,target2, self.request_writes,self.request_observed,override_active[0]); Ok(()) } pub(crate) fn render_pb(&mut self) -> Result { - self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3_render::PB_PAGES).map_err(|_|EINVAL)?) + self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3::m3_render::PB_PAGES).map_err(|_|EINVAL)?) } pub(crate) fn submit_queue(&mut self, kind:usize, queue:u64, head:u16,event:u8,new:bool)->Result { if kind>2 {return Err(EINVAL);} @@ -316,7 +267,7 @@ impl Config { Ok(self.objects[RUNTIME_POINTERS].read_u32(0x314)? == 0xabcdabcd && self.objects[RUNTIME_FLAGS].read_u32(4)? == 1) } - pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDrmDevice) -> Result { let magic=self.objects[RUNTIME_POINTERS].read_u32(0x314)?; let state=self.objects[RUNTIME_POINTERS].read_u32(0x2fc)?; let a=self.objects[RUNTIME_FLAGS].read_u32(4)?; @@ -345,7 +296,7 @@ impl Config { /// at +0x4580/+0x4590/+0x45a0, the firmware's own recovery information /// at +0x40a0, and the details it leaves for a recovery event at +0x44a0. /// Nothing is acknowledged, cleared or resumed here. - pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDevice, events: u64) -> Result { + pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDrmDevice, events: u64) -> Result { let control=&mut self.objects[CONTROL_REGION]; let halt_count=control.read_u64(0x4580)?; let halted=control.read_u32(0x4590)?; @@ -360,7 +311,7 @@ impl Config { /// Read-only: log the event ring entries from the host's read index to the firmware's write /// index (at most 16), as `drain` reads them. Nothing is consumed or acknowledged, and the /// completion count is not changed. - pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDrmDevice) -> Result { const SIZE: usize = 0x38; const COUNT: usize = 256; let r = self.objects[storage::EVENT*2].read_u32(0)? as usize; @@ -395,7 +346,7 @@ impl Config { pub(crate) fn control_done(&mut self, next: u32) -> Result { Ok(self.objects[storage::DEVICE_CONTROL*2].read_u32(0)? == next && self.objects[storage::DEVICE_CONTROL*2].read_u32(0x10)? == next) } - pub(crate) fn drain(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn drain(&mut self, dev: &driver::AsahiDrmDevice) -> Result { // Event, trace, stats and six independent firmware log rings. for (state, ring, size, count, channels) in [(storage::EVENT*2,storage::EVENT*2+1,0x38,256,1), (storage::TRACE*2,storage::TRACE*2+1,0x38,512,1), diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index 1c2f0b1e8866d..c43017a8933c0 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -12,6 +12,7 @@ use kernel::{ io::{ mem::IoMem, Io, + IoBase, }, platform, prelude::*, @@ -19,16 +20,16 @@ use kernel::{ time::{delay::fsleep, Delta, Instant, Monotonic}, }; -use crate::m3_firmware::Firmware; -use crate::m3_soc::Soc; +use crate::m3::m3_firmware::Firmware; +use crate::m3::m3_soc::Soc; const ASC_CPU_CONTROL: usize = 0x44; const ASC_CPU_RUN: u32 = 1 << 4; pub(crate) struct Device { dev: ARef, - asc: Pin>>>, - sgx: Pin>>, + asc: Devres>, + sgx: Devres>, firmware: Firmware, core_mask: u64, soc: &'static Soc, @@ -37,34 +38,28 @@ pub(crate) struct Device { impl Device { /// `experiment` is the T8122 start experiment's values, on an armed T8122 only: it decides - /// the SGX setup write, which the T8122 table lacks. `t6031` is the same for an armed T6031. - pub(crate) fn new(pdev: &platform::Device, firmware: Firmware, soc: &'static Soc, - experiment: Option<&crate::t8122_start::Experiment>, - t6031: Option<&crate::t6031_start::Experiment>) -> Result { + /// the SGX setup write, which the T8122 table lacks. + pub(crate) fn new(pdev: &platform::Device>, firmware: Firmware, soc: &'static Soc, + experiment: Option<&crate::m3::t8122_start::Experiment>) -> Result { // Map ASC before taking a vote. The larger SGX aperture contains // this control window and the separate mailbox provider, so it is // mapped without a conflicting claim over those child resources. // Only the 16 KiB control window is used. Some device trees describe // the ASC block as a larger range that also covers the mailbox, which // the mailbox driver has already claimed, so do not claim the range. - let asc = KBox::pin_init( - pdev.io_request_by_name(c_str!("asc")) - .ok_or(EINVAL)? - .iomap_sized::<0x4000>(), - GFP_KERNEL, - )?; + let asc = pdev.io_request_by_name(c_str!("asc")) + .ok_or(EINVAL)? + .iomap_sized::<0x4000>()? + .into_devres()?; dev_info!(pdev.as_ref(), "M3: ASC aperture mapped\n"); - let sgx = KBox::pin_init( - pdev.io_request_by_name(c_str!("sgx")) - .ok_or(EINVAL)? - .iomap(), - GFP_KERNEL, - )?; + let sgx = pdev.io_request_by_name(c_str!("sgx")) + .ok_or(EINVAL)? + .iomap()? + .into_devres()?; dev_info!(pdev.as_ref(), "M3: SGX aperture mapped\n"); - let pmp = crate::m3_board::has_pmp_link(pdev); - let t8122 = crate::t8122_start::is_t8122(soc); - let t6031_soc = crate::t6031_start::is_t6031(soc); - if pmp && !t8122 && !t6031_soc { + let pmp = crate::m3::m3_board::has_pmp_link(pdev); + let t8122 = crate::m3::t8122_start::is_t8122(soc); + if pmp && !t8122 { dev_err!(pdev.as_ref(), "M3: optional apple,pmp GPU link is unsupported\n"); return Err(ENOTSUPP); } @@ -78,7 +73,7 @@ impl Device { } else { dev_info!(pdev.as_ref(), "M3: no apple,pmp link; GPU power is left to its power domain\n"); } - if asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + if asc.access(pdev.as_ref())?.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { dev_err!( pdev.as_ref(), "M3 {}: ASC already running; refusing to take ownership\n", @@ -103,19 +98,15 @@ impl Device { // failed acquisition releases it only after the ASC-stopped check in Drop. device.power_vote = true; to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 1) })?; - if t6031_soc { - dev_info!(pdev.as_ref(), "M3 G15C: PMP GPU device 5 power acknowledged\n"); - } else { - dev_info!(pdev.as_ref(), "M3 G15G: PMP GPU device 5 power acknowledged\n"); - } + dev_info!(pdev.as_ref(), "M3 G15G: PMP GPU device 5 power acknowledged\n"); } - let registers = device.sgx.access(pdev.as_ref())?; + let registers = device.sgx.access(pdev.as_ref())?.as_view(); let version = registers.try_read32(0xd04000)?; let counts = registers.try_read32(0xd04010)?; // The core-enable words: the second only on a SoC with more than 32 core slots. let mut core_mask = u64::from(registers.try_read32(0xe01500)?); - if crate::m3_board::core_slots(soc) > 32 { + if crate::m3::m3_board::core_slots(soc) > 32 { core_mask |= u64::from(registers.try_read32(0xe01504)?) << 32; } dev_info!(pdev.as_ref(), "M3 {}: power acknowledged, version={:#010x} counts={:#010x} core-mask={:#x}, firmware={}\n", @@ -124,7 +115,7 @@ impl Device { // fused core mask varies with SKU; absent cores must never be enabled. if version & soc.id.version_mask != soc.id.version || counts & soc.id.counts_mask != soc.id.counts - || !crate::m3_board::core_mask_valid(soc, core_mask) + || !crate::m3::m3_board::core_mask_valid(soc, core_mask) { return Err(ENODEV); } @@ -138,21 +129,7 @@ impl Device { } e.sgx_setup() } - None => match t6031 { - Some(e) => { - match e.sgx_setup() { - Some((offset, value)) => dev_warn!( - pdev.as_ref(), - "M3 G15C start: SGX setup write: SGX+{:#x} = {:#x}\n", - offset, - value - ), - None => dev_info!(pdev.as_ref(), "M3 G15C start: no SGX setup write\n"), - } - e.sgx_setup() - } - None => Some(soc.sgx_setup.ok_or(ENODEV)?), - }, + None => Some(soc.sgx_setup.ok_or(ENODEV)?), }; if let Some((setup_offset, setup_value)) = setup { registers.try_write32(setup_value, setup_offset)?; @@ -173,7 +150,8 @@ impl Device { /// accesses selectable (`asahi.m3_retire_mmio`); with neither it touches no GPU register. pub(crate) fn check_idle_parts(&self,busy:bool,faults:bool)->Result { if !busy && !faults {return Ok(());} - let sgx=self.sgx.try_access().ok_or(ENODEV)?; + let sgx_guard=self.sgx.try_access().ok_or(ENODEV)?; + let sgx=sgx_guard.as_view(); if busy && (sgx.try_read64(0xc020)? | sgx.try_read64(0xc120)?) & 1 != 0 { return Err(EBUSY); } @@ -190,34 +168,16 @@ impl Device { if fault!=0 {return Err(EIO);} Ok(()) } - pub(crate) fn check_drm(&self, pdev: &platform::Device) -> Result { - use kernel::dma::{Device as _, DmaMask}; - // SAFETY: This is the admitted M3 GPU's 42-bit physical DMA capability. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; - crate::mmu::check_handoff_guard()?; - dev_info!(pdev.as_ref(), "M3 {}: GPU handoff lock checked.\n", self.soc.gpu_name); - for _ in 0..2 { - let drm: ARef = kernel::drm::Device::new( - pdev.as_ref(), crate::driver::AsahiData::new(pdev, None, true))?; - if drm.gpu().is_ok() { return Err(EIO); } - let mut object = crate::gem::new_kernel_object(&drm, 0x4000)?; - object.vmap()?.memset(0); - drop(object); - drop(drm); - } - dev_info!(pdev.as_ref(), "M3 {}: DRM data initialized; backend absent, device unregistered\n", self.soc.gpu_name); - Ok(()) - } - /// The raw performance-state register (SGX+0xe01000). pub(crate) fn pstate_register(&self) -> Result { - self.sgx.try_access().ok_or(ENODEV)?.try_read32(0xe01000) + self.sgx.try_access().ok_or(ENODEV)?.as_view().try_read32(0xe01000) } /// Read-only snapshot. Do not select or acknowledge fault banks here; /// the firmware owns those registers until recovery is ported. pub(crate) fn log_engine_state(&self, vm: &crate::mmu::Vm) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx_guard = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = sgx_guard.as_view(); let pstate = sgx.try_read32(0xe01000)? & 0xf; if pstate == 0 { dev_info!(self.dev.as_ref(), "M3 {}: GPU is powered down; skipping engine/MMU register snapshot\n", self.soc.gpu_name); @@ -240,10 +200,10 @@ impl Device { let service=sgx.try_read32(0xa010)?; let debug=sgx.try_read32(0xa000)?; let mut cores=u64::from(sgx.try_read32(0xe01500)?); - if crate::m3_board::core_slots(self.soc) > 32 { + if crate::m3::m3_board::core_slots(self.soc) > 32 { cores |= u64::from(sgx.try_read32(0xe01504)?) << 32; } - for core in 0..crate::m3_board::core_slots(self.soc).min(64) { + for core in 0..crate::m3::m3_board::core_slots(self.soc).min(64) { if cores & (1u64< &Firmware { &self.firmware } - pub(crate) fn require_stopped(&self, pdev: &platform::Device) -> Result { - if self.asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + pub(crate) fn require_stopped(&self, pdev: &platform::Device>) -> Result { + if self.asc.access(pdev.as_ref())?.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { return Err(EBUSY); } Ok(()) } - pub(crate) fn start_asc(&self, pdev: &platform::Device) -> Result { + pub(crate) fn start_asc(&self, pdev: &platform::Device>) -> Result { self.require_stopped(pdev)?; - let asc = self.asc.access(pdev.as_ref())?; + let asc = self.asc.access(pdev.as_ref())?.as_view(); asc.write32(asc.read32(ASC_CPU_CONTROL) | ASC_CPU_RUN, ASC_CPU_CONTROL); Ok(()) } pub(crate) fn stop_asc(&self) -> Result { - let asc = self.asc.try_access().ok_or(ENODEV)?; + let asc_guard = self.asc.try_access().ok_or(ENODEV)?; + let asc = asc_guard.as_view(); asc.write32(asc.read32(ASC_CPU_CONTROL) & !ASC_CPU_RUN, ASC_CPU_CONTROL); let start = Instant::::now(); while asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { @@ -301,7 +262,7 @@ impl Drop for Device { } // ASC and firmware queues must be stopped before the inner GPU vote is relinquished. match self.asc.try_access() { - Some(asc) if asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN == 0 => {} + Some(asc) if asc.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN == 0 => {} _ => { dev_err!(self.dev.as_ref(), "M3 G15G: cannot prove ASC stopped; retaining PMP vote\n"); return; diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 25e65c6599084..2d93823e67d01 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! M3 through the common Asahi GEM, GPUVM and render-node frontend. use core::ops::Range; -use kernel::{device::Core, drm, new_mutex, platform, prelude::*, sync::{Arc, Mutex}}; -use crate::{alloc, driver, drm_gpu::{DrmGpu, DrmGpuParams}, gem, gpu, hw, mmu, queue}; +use core::any::Any; +use kernel::{device::Core, drm, new_mutex, platform, prelude::*, sync::{Arc, Mutex}, uapi}; +use crate::{alloc, driver, gem, gpu, m3::hw, mmu, queue}; -pub(crate) type Shared = Arc>>; -struct ProgressView(Arc); +pub(crate) type Shared = Arc>>; +struct ProgressView(Arc); impl kernel::debugfs::Writer for ProgressView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (generation, completed, last_ns, healthy) = self.0.progress_snapshot(); @@ -14,7 +15,7 @@ impl kernel::debugfs::Writer for ProgressView { } } // Separate endpoint preserves the strict completion-v1 ABI for old guards. -struct ActivityView(Arc); +struct ActivityView(Arc); impl kernel::debugfs::Writer for ActivityView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (generation, epoch, healthy) = self.0.activity_snapshot(); @@ -23,7 +24,7 @@ impl kernel::debugfs::Writer for ActivityView { } } -struct TimingView(Arc); +struct TimingView(Arc); impl kernel::debugfs::Writer for TimingView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (generation, _, _, healthy) = self.0.progress_snapshot(); @@ -33,7 +34,7 @@ impl kernel::debugfs::Writer for TimingView { // Root-only writable debugfs policy; no mutable module-parameter references. // Keep ownership independent of the runtime lock and remove the file on stop. -struct CompletionView(Arc); +struct CompletionView(Arc); impl kernel::debugfs::Writer for CompletionView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); @@ -53,7 +54,7 @@ impl kernel::debugfs::Reader for CompletionView { } } -struct SubmitOverlapView(Arc); +struct SubmitOverlapView(Arc); impl kernel::debugfs::Writer for SubmitOverlapView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { writeln!(f, "{}", u8::from(self.0.cpu_overlap())) @@ -74,137 +75,179 @@ impl kernel::debugfs::Reader for SubmitOverlapView { } } -pub(crate) struct Registered { - registration: Pin>>>>, +/// The started T8122 runtime, as the shared DRM layer sees it ([`gpu::Gpu`]). It is created at +/// probe with the runtime it wraps and handed to the DRM registration, which owns it: dropping it +/// (unbind, or a failed registration) stops the runtime. The debugfs files share its lifetime. +pub(crate) struct Backend { shared: Shared, - health: Arc, + health: Arc, + ids: gpu::SequenceIDs, + scheduler: Arc>, + core_mask: u64, + max_frequency_khz: u32, + soc: &'static crate::m3::m3_soc::Soc, _progress: Pin>>, _activity: Pin>>, - _memory: Pin>>, + _memory: Pin>>, _timing: Pin>>, _completion: Pin>>, _overlap: Pin>>, } -impl Registered { - pub(crate) fn start(pdev: &platform::Device, soc: &'static crate::m3_soc::Soc) -> Result { - let resources = crate::m3_resources::from_device(pdev, soc).inspect_err(|_| { - crate::t8122_start::not_admitted(pdev.as_ref(), soc); - crate::t6031_start::not_admitted(pdev.as_ref(), soc); + +impl Backend { + /// Admits and starts the M3 runtime on `soc` (the T8122), with the DRM device `drm` that the + /// caller registers afterwards. + pub(crate) fn start( + pdev: &platform::Device>, + drm: &driver::AsahiDrmDevice, + soc: &'static crate::m3::m3_soc::Soc, + ) -> Result> { + // The runtime only exists to back its render node: without one (`asahi.m3_expose=0`) + // nothing is started, rather than leaving a running firmware behind a failed probe. + if !crate::m3::m3_board::expose_render_node(soc) { + dev_info!(pdev.as_ref(), "M3: render node not registered on this board (asahi.m3_expose=1 registers it); GPU not started\n"); + return Err(ENODEV); + } + let resources = crate::m3::m3_resources::from_device(pdev, soc).inspect_err(|_| { + crate::m3::t8122_start::not_admitted(pdev.as_ref(), soc); })?; dev_info!(pdev.as_ref(), "M3: resource admission complete\n"); - // An armed T8122 or T6031 start supplies the values its table leaves out. Any other SoC, - // or an unarmed T8122 or T6031, needs a complete table. - let experiment = crate::t8122_start::arm(pdev.as_ref(), soc)?; - let t6031 = crate::t6031_start::arm(pdev.as_ref(), soc)?; - if experiment.is_none() && t6031.is_none() { + // An armed T8122 start supplies the values its table leaves out. An unarmed T8122 + // needs a complete table, which it does not have: it is refused here. + let experiment = crate::m3::t8122_start::arm(pdev.as_ref(), soc)?; + if experiment.is_none() { soc.require_complete(pdev.as_ref())?; } let experiment = experiment.as_ref(); - let t6031 = t6031.as_ref(); let refused = |stage: &'static str| { - move |e: &Error| { - crate::t8122_start::refused(pdev.as_ref(), experiment, stage, *e); - crate::t6031_start::refused(pdev.as_ref(), t6031, stage, *e); - } + move |e: &Error| crate::m3::t8122_start::refused(pdev.as_ref(), experiment, stage, *e) }; - let firmware = crate::m3_firmware::identify_loaded(pdev, soc, resources, experiment, t6031) + let firmware = crate::m3::m3_firmware::identify_loaded(pdev, soc, resources, experiment) .inspect_err(refused("the firmware identity"))?; dev_info!(pdev.as_ref(), "M3: firmware identity accepted\n"); // InitData source and contents, before any GPU register access. - let contents = crate::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment, t6031) + let contents = crate::m3::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment) .inspect_err(refused("the InitData contents"))?; - let device = crate::m3_device::Device::new(pdev, firmware, soc, experiment, t6031) + let device = crate::m3::m3_device::Device::new(pdev, firmware, soc, experiment) .inspect_err(refused("the GPU registers and identity"))?; - device.check_drm(pdev).inspect_err(refused("the DRM setup"))?; let core_mask = device.core_mask(); let max_frequency_khz = 1000 * contents.pstates.reported_max_mhz(); - let mut runtime = crate::m3_runtime::Runtime::new(pdev, device, contents)?; + let mut runtime = crate::m3::m3_runtime::Runtime::new(pdev, drm, device, contents)?; runtime.boot(pdev)?; dev_info!(pdev.as_ref(), "M3: runtime switches: asahi.m3_unlocked_wait={} asahi.m3_timeout_nohang={} resume-after-fault={}\n", - u8::from(crate::m3_params::unlocked_wait()), u8::from(crate::m3_params::timeout_nohang()), - u8::from(crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault))); - let drm = runtime.drm(); + u8::from(crate::m3::m3_params::unlocked_wait()), u8::from(crate::m3::m3_params::timeout_nohang()), + u8::from(crate::m3::m3_params::g15_debug(crate::m3::m3_params::G15Debug::M3ResumeAfterFault))); let health = runtime.health(); - // Health holds no device references. The read-only file owns an Arc - // and is removed with registration; teardown latches failure first. + let state = runtime.rtkit_state(); + // Health holds no device references. The read-only files own an Arc and are removed + // with the backend; teardown latches failure first. let directory = kernel::debugfs::Dir::new(c"asahi-m3"); let progress = KBox::pin_init(directory.read_only_file(c"progress", ProgressView(health.clone())), GFP_KERNEL)?; let activity = KBox::pin_init(directory.read_only_file(c"activity", ActivityView(health.clone())), GFP_KERNEL)?; let memory = KBox::pin_init(directory.read_only_file(c"memory", - crate::agx_memory_stats::View), GFP_KERNEL)?; + crate::m3::agx_memory_stats::View), GFP_KERNEL)?; let timing = KBox::pin_init(directory.read_only_file(c"timing", TimingView(health.clone())), GFP_KERNEL)?; let completion = KBox::pin_init(directory.read_write_file(c"completion_wait", CompletionView(runtime.completion_wait())), GFP_KERNEL)?; let overlap = KBox::pin_init(directory.read_write_file(c"submit_overlap", SubmitOverlapView(runtime.completion_wait())), GFP_KERNEL)?; - let shared = Arc::pin_init(new_mutex!(Some(runtime)), GFP_KERNEL)?; - if !drm.completion.populate(crate::m3_completion::Completion::new(shared.clone())?) { + let scheduler = Arc::new(drm::sched::Scheduler::new(drm.as_ref().as_ref(), 4, 8, 0, 3000, + kernel::c_str!("asahi_m3_sched"))?, GFP_KERNEL)?; + let shared: Shared = Arc::pin_init(new_mutex!(Some(runtime)), GFP_KERNEL)?; + let completion_work = crate::m3::m3_completion::Completion::new(shared.clone())?; + // From here on every failure drops the backend, which stops the runtime. + let backend = Arc::new(Self { shared, health, ids: gpu::SequenceIDs::default(), + scheduler, core_mask, max_frequency_khz, soc, + _progress: progress, _activity: activity, _memory: memory, _timing: timing, + _completion: completion, _overlap: overlap }, GFP_KERNEL)?; + // The completion worker services this runtime from now on; the RTKit callbacks and + // the submission path queue it through the runtime's state. + if !state.completion.populate(completion_work) { return Err(EBUSY); } - let owner = Self { registration: KBox::pin_init(new_mutex!(None), GFP_KERNEL)?, shared: shared.clone(), health: health.clone(), _progress: progress, _activity: activity, _memory: memory, _timing: timing, _completion: completion, _overlap: overlap }; - let scheduler=Arc::new(drm::sched::Scheduler::new(drm.as_ref(),4,8,0,3000,kernel::c_str!("asahi_m3_sched"))?,GFP_KERNEL)?; - let backend: Arc = Arc::new(Backend { shared, health, scheduler, ids: gpu::SequenceIDs::default(), - core_mask, max_frequency_khz, soc }, GFP_KERNEL)?; - if !drm.gpu.populate(crate::drm_gpu::Backend::M3(backend)) { return Err(EBUSY); } - if crate::m3_board::expose_render_node(soc) { - *owner.registration.lock() = Some(drm::driver::Registration::new(&drm, 0)?); - dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); - } else { - dev_info!(pdev.as_ref(), "M3: firmware ready; render node not registered on this board (asahi.m3_expose=1 registers it)\n"); - } - Ok(owner) + Ok(backend) } + + /// Stops the runtime: called on unbind, while the device's resources are still live, and + /// again (doing nothing) when the backend is dropped. pub(crate) fn stop(&self) { self.health.mark_failed(); - drop(self.registration.lock().take()); drop(self.shared.lock().take()); } } -impl Drop for Registered { fn drop(&mut self) { self.stop(); } } -struct Backend { shared: Shared, health: Arc, ids: gpu::SequenceIDs, - scheduler:Arc>, core_mask: u64, max_frequency_khz: u32, - soc: &'static crate::m3_soc::Soc } -impl DrmGpu for Backend { + +impl Drop for Backend { + fn drop(&mut self) { + self.stop(); + } +} + +impl gpu::Gpu for Backend { + fn stats_snapshot(&self) -> Option> { None } + fn as_any(&self) -> &dyn Any { self } + fn arc_as_any(self: Arc) -> Arc { self as Arc } + fn manager(&self) -> Option<&dyn gpu::GpuManager> { None } fn init(&self) -> Result { if self.is_crashed() { Err(ENODEV) } else { Ok(()) } } fn ids(&self) -> &gpu::SequenceIDs { &self.ids } fn is_crashed(&self) -> bool { !self.health.healthy() } - fn update_globals(&self) {} - - fn supports_vm_status(&self) -> bool { true } - fn supports_scheduled_queues(&self)->bool {true} - fn submission_error(&self) -> i32 { if self.is_crashed() { EIO.to_errno() } else { 0 } } - fn params(&self) -> Result { + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { let soc = self.soc; - let masks = crate::m3_board::core_masks(soc, self.core_mask); - Ok(DrmGpuParams { gpu_generation: hw::GpuGen::G15 as u32, - gpu_variant: soc.gpu_variant as u32, gpu_revision: soc.gpu_revision as u32, - chip_id: soc.chip_id, num_dies: 1, num_clusters_total: soc.clusters, - num_cores_per_cluster: soc.cores_per_cluster, - core_masks: masks, max_frequency_khz: self.max_frequency_khz, usc_generation: 3, - gpu_hal_generation: hw::GpuHalGeneration::Legacy as u32, - max_commands_per_submission: crate::file::MAX_COMMANDS_PER_SUBMISSION }) - } - fn user_range(&self) -> Result> { Ok(mmu::UAT_PGSZ as u64..0x2ff_ffff_8000) } - fn unknown_page(&self) -> Result { Ok(0x2ff_ffff_8000) } + let masks = crate::m3::m3_board::core_masks(soc, self.core_mask); + params.gpu_generation = hw::GpuGen::G15 as u32; + params.gpu_variant = soc.gpu_variant as u32; + params.gpu_revision = soc.gpu_revision as u32; + params.chip_id = soc.chip_id; + params.num_dies = 1; + params.num_clusters_total = soc.clusters; + params.num_cores_per_cluster = soc.cores_per_cluster; + for (out, mask) in params.core_masks.iter_mut().zip(masks.iter()) { + *out = (*mask).into(); + } + params.max_frequency_khz = self.max_frequency_khz; + params.usc_generation = 3; + params.gpu_hal_generation = hw::GpuHalGeneration::Legacy as u32; + Ok(()) + } + fn supports_vm_status(&self) -> bool { true } + /// The features the 7.1.12 file layer reported for the M3 runtime: VM status (not its + /// mirror), scheduled queues, and the SoC's optional render and compute features. + fn feature_flags(&self) -> u64 { + let mut flags = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SCHEDULED_QUEUES as u64; + if self.soc.features.fragment_dependency && crate::m3::m3_params::early_tiling() { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY as u64; + } + if self.soc.features.compute_wide_visibility { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY as u64; + } + flags + } fn base_clock_hz(&self) -> u32 { let value: u64; // SAFETY: read-only architectural counter frequency. unsafe { core::arch::asm!("mrs {x}, CNTFRQ_EL0", x=out(reg) value) }; value as u32 } + fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } + fn user_range(&self) -> Range { mmu::UAT_PGSZ as u64..0x2ff_ffff_8000 } + fn tracked_submission(&self) -> bool { true } + /// Only M3-aware userspace may create a VM (7.1.12's per-file client gate). + fn admit_vm(&self, dev: &kernel::device::Device, params_size: u64, flags: u32) -> Result { + crate::m3::m3_client::admit_vm(dev, params_size, flags) + } fn new_vm(&self, range: Range) -> Result { Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.new_vm(self.ids.vm.next(), range) } fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64) -> Result> { - Ok(KBox::new(crate::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, + Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, self.soc.features.fragment_dependency)?,GFP_KERNEL)?) } fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { - self.shared.lock().as_ref().ok_or(ENODEV)?.map_timestamp(bo,range) + Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index 9ae509ad959c4..0d2835174c677 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -use crate::m3_resources as agx_resources; +use crate::m3::m3_resources as agx_resources; /// A GPU firmware image the runtime can identify. #[derive(Debug)] @@ -30,7 +30,7 @@ pub(crate) static KNOWN_IMAGES: [KnownImage; 2] = [ KnownImage { name: "J514S RTKit-2419.140.12", uuid: None, - stkg_sha256: Some(crate::m3_firmware::TEXT_SHA256), + stkg_sha256: Some(crate::m3::m3_firmware::TEXT_SHA256), initdata_magic: Some(G15_V14_8_3_INITDATA), }, KnownImage { @@ -140,20 +140,6 @@ pub(crate) static T8122_LAYOUT: Layout = Layout { version: "RTKit-2419.140.12.release", }; -/// The T6031 firmware-compat 14.8.3 segment sizes and VAs, from the J516C ADT (iop-gfx-nub0 -/// segment-ranges: TEXT 0x64000 and DATA 0x140000 at the VAs both other layouts use). Neither -/// existing layout matches them. The boot-entropy tag offset is not known (0 stands for it); -/// an armed start searches the loaded text for the tag. The loaded image's version string is -/// not known ("unknown" stands for it). -pub(crate) const T6031_LAYOUT: Layout = Layout { - text_size: 0x64000, - data_size: 0x14_0000, - vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], - entropy_tag_offset: 0, - entropy_tag: *b"GKTS\x08\x00\x00\x00", - version: "unknown", -}; - pub(crate) const TEXT_SHA256: [u8; 32] = [ 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; @@ -194,11 +180,10 @@ impl Firmware { #[cfg(not(test))] pub(crate) fn identify_loaded( - pdev: &kernel::platform::Device, - soc: &'static crate::m3_soc::Soc, + pdev: &kernel::platform::Device>, + soc: &'static crate::m3::m3_soc::Soc, resources: agx_resources::Resources, - experiment: Option<&crate::t8122_start::Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, + experiment: Option<&crate::m3::t8122_start::Experiment>, ) -> kernel::error::Result { use kernel::{ bindings, c_str, @@ -220,21 +205,6 @@ pub(crate) fn identify_loaded( let bytes = unsafe { core::slice::from_raw_parts(mapping.ptr(), mapping.size()) }; let mut canonical = KVec::new(); canonical.extend_from_slice(bytes, GFP_KERNEL)?; - // An armed T6031 searches the loaded text for its boot-entropy tag. The layout's offset is - // a placeholder, so the T8122 image table must not see this image. - if let Some(experiment) = t6031 { - experiment.normalize_text(&mut canonical)?; - let mut digest = [0u8; 32]; - // SAFETY: the initialized private copy and distinct digest live for the synchronous - // SHA-256 call. Normalization never writes the loaded firmware. - unsafe { bindings::sha256(canonical.as_ptr(), canonical.len(), digest.as_mut_ptr()) }; - experiment.admit_image(pdev.as_ref(), bytes, &digest)?; - return Ok(Firmware { - resources, - initdata_magic: experiment.initdata_version(), - layout, - }); - } if !normalize_boot_entropy(layout, &mut canonical) { return Err(ENODEV); } @@ -247,7 +217,7 @@ pub(crate) fn identify_loaded( Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..], None => soc.images, }; - let image = crate::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; + let image = crate::m3::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; let initdata_magic = match experiment { Some(experiment) => experiment.initdata_version(), None => image.initdata_magic.ok_or(ENODEV)?, @@ -393,21 +363,6 @@ mod tests { } } - #[test] - fn t6031_layout_is_the_adt_segments_and_matches_neither_other_layout() { - let layout = &T6031_LAYOUT; - assert_eq!((layout.text_size, layout.data_size), (0x64000, 0x140000)); - assert_eq!(layout.vas, [0xffff_fc00_0000_0000, 0xffff_fc00_0000_0000 + 0x64000]); - assert_eq!(layout.entropy_tag, T8122_LAYOUT.entropy_tag); - let mut regions = [Region { base: 0, size: 0 }; 6]; - regions[4].size = layout.text_size; - regions[5].size = layout.data_size; - let resources = Resources { regions, firmware_vas: layout.vas }; - assert!(Firmware::matching_layout(layout, &resources)); - assert!(!Firmware::matching_layout(&T6030_LAYOUT, &resources)); - assert!(!Firmware::matching_layout(&T8122_LAYOUT, &resources)); - } - #[test] fn entropy_normalization_changes_only_the_eight_boot_bytes() { for layout in [&T6030_LAYOUT, &T8122_LAYOUT] { diff --git a/drivers/gpu/drm/asahi/m3/m3_memory.rs b/drivers/gpu/drm/asahi/m3/m3_memory.rs index cbfb771fd0873..5e8018a802e70 100644 --- a/drivers/gpu/drm/asahi/m3/m3_memory.rs +++ b/drivers/gpu/drm/asahi/m3/m3_memory.rs @@ -8,6 +8,20 @@ enum Backing { Paged(gem::ObjectRef), } +/// Invalidates context 0's non-global translations when dropped. A job's command aliases live in +/// the context-0 lower tree, which has no bind slot on the adopted UAT (7.1.12 gave it the fixed +/// slot 0), so unmapping them invalidates nothing by itself. Declared first in a constructor and +/// last in the job, it drops after every alias. +pub(crate) struct LowerTlbi; + +impl Drop for LowerTlbi { + fn drop(&mut self) { + crate::mem::sync(); + crate::mem::tlbi_asid(0); + crate::mem::sync(); + } +} + pub(crate) struct Buffer { mapping: mmu::KernelMapping, gpu_mapping: Option, @@ -17,18 +31,31 @@ pub(crate) struct Buffer { object: Backing, offset: usize, size: usize, - _coherent_allocation: Option, + _coherent_allocation: Option, +} + +impl Drop for Buffer { + fn drop(&mut self) { + // A GPU view may be a command alias in the context-0 lower tree, which has no bind slot + // to invalidate on unmap: drop it and invalidate context 0 before the backing below is + // freed (7.1.12 invalidated inside the unmap). + if self.gpu_mapping.take().is_some() { + crate::mem::sync(); + crate::mem::tlbi_asid(0); + crate::mem::sync(); + } + } } impl Buffer { - pub(crate) fn at(dev: &driver::AsahiDevice, fw: &mmu::Vm, + pub(crate) fn at(dev: &driver::AsahiDrmDevice, fw: &mmu::Vm, gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, gpu_shared: bool) -> Result { Self::at_prot(dev,fw,gpu,address,size, if gpu_shared {prot::PROT_GPU_FW_SHARED_RW} else {prot::PROT_FW_SHARED_RW}, prot::PROT_GPU_SHARED_RW) } - pub(crate) fn at_prot(dev: &driver::AsahiDevice, fw: &mmu::Vm, + pub(crate) fn at_prot(dev: &driver::AsahiDrmDevice, fw: &mmu::Vm, gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, protection: prot::Prot, gpu_protection: prot::Prot) -> Result { let offset=(address.or(gpu.map(|(_,a)|a)).unwrap_or(0)&0x3fff) as usize; @@ -39,18 +66,18 @@ impl Buffer { // normal case here and is not reported; only a failed fallback is. // SAFETY: M3 runtime owns the bound platform device for the entire // diagnostic; all mappings retire before normal DMA allocation Drop. - let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_bound()},allocated,GFP_KERNEL, + let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_ref().as_bound()},allocated,GFP_KERNEL, kernel::dma::attrs::DMA_ATTR_NO_WARN).inspect_err(|_| dev_err!(dev.as_ref(),"M3: cannot allocate {} KiB of GPU memory\n",allocated/1024))?; let pa:usize=object.dma_handle().try_into()?; - let mapping=if let Some(a)=address {fw.map_io(a&!0x3fff,pa,allocated,protection)?} + let mapping=if let Some(a)=address {fw.map_io_m3(a&!0x3fff,pa,allocated,protection)?} else {fw.map_io_in_range(0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000,pa,allocated,protection)?}; - let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; + let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io_m3(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Coherent(object),offset,size, - _coherent_allocation:Some(crate::agx_memory_stats::Allocation::coherent(allocated)?)}) + _coherent_allocation:Some(crate::m3::agx_memory_stats::Allocation::coherent(allocated)?)}) } /// Allocate owned tiler storage in the driver-reserved GPU range. Keep the /// old allocation mapped until the replacement has been fully allocated. - pub(crate) fn tiler(dev:&driver::AsahiDevice,fw:&mmu::Vm,vm:&mmu::Vm,size:usize)->Result { + pub(crate) fn tiler(dev:&driver::AsahiDrmDevice,fw:&mmu::Vm,vm:&mmu::Vm,size:usize)->Result { if size==0 || size.checked_add(mmu::UAT_PGMSK).is_none() {return Err(EINVAL);} // Large render targets can require 128 MiB of tiler storage. Engines // and firmware address it through UAT, so physically contiguous DMA @@ -62,7 +89,7 @@ impl Buffer { mmu::UAT_PGSZ as u64,prot::PROT_GPU_FW_SHARED_RW,true)?; // Growth retains the cached GPU-only policy of the original TPC and // tilemap allocations; the firmware alias remains shared/uncached. - let gpu_protection=prot::Prot::from_pte(crate::m3_pass_layout::Access::GpuCachedRw.pte()); + let gpu_protection=prot::Prot::from_pte(crate::m3::m3_pass_layout::Access::GpuCachedRw.pte()); let gpu_mapping=Some(object.map_into_range(vm,0x74_0000_0000..0x78_0000_0000, mmu::UAT_PGSZ as u64,gpu_protection,true)?); Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Paged(object),offset:0,size,_coherent_allocation:None}) @@ -73,7 +100,7 @@ impl Buffer { pub(crate) fn map_gpu_view(&mut self,vm:&mmu::Vm)->Result { let address=self.gpu_va()?&!0x3fff; match &mut self.object { - Backing::Coherent(object)=>vm.map_io(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), + Backing::Coherent(object)=>vm.map_io_m3(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), Backing::Paged(object)=>object.map_at(vm,address,self.gpu_protection,true), } } @@ -82,7 +109,10 @@ impl Buffer { fn cpu_ptr(&mut self)->Result<*mut u8> { match &mut self.object { Backing::Coherent(object)=>Ok(object.as_mut_ptr().cast::()), - Backing::Paged(object)=>Ok(object.cached_vmap()?.as_mut_ptr()), + Backing::Paged(object)=>{ + use kernel::io::IoBase; + Ok(object.vmap()?.as_view().as_ptr().cast::()) + } } } @@ -124,7 +154,7 @@ impl Buffer { if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} let ptr=self.cpu_ptr()?; // SAFETY: checked span in retained CPU storage; no aligned access is required. - unsafe {crate::m3_consistent_perf::store_packed(ptr.add(self.offset+offset),value)}; + unsafe {crate::m3::m3_consistent_perf::store_packed(ptr.add(self.offset+offset),value)}; Ok(()) } pub(crate) fn read(&mut self,offset:usize,data:&mut[u8])->Result { diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs index d34e1f23840eb..8167c6c3ec05c 100644 --- a/drivers/gpu/drm/asahi/m3/m3_params.rs +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -139,7 +139,7 @@ unsafe extern "C" fn get_atomic_param( } fn parse_render_batch_override(text: &str) -> Option { - parse_u64(text).filter(|value| *value <= crate::m3_pass_layout::SLOTS as u64) + parse_u64(text).filter(|value| *value <= crate::m3::m3_pass_layout::SLOTS as u64) } /// Runtime tuning is opt-in: zero keeps the existing boot parameter and its @@ -202,20 +202,58 @@ pub(crate) fn render_internal_timestamps()->bool { M3_RENDER_INTERNAL_TIMESTAMPS.load(Ordering::Relaxed)!=0 } +// The runtime's boot-time module parameters. On 7.1.12 they were `module!` parameters; they +// keep their names, and exist only with the T8122 runtime built in. + +/// `asahi.m3_render_batch_size`: bounded ordered render batch size (1..16). +static M3_RENDER_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_render_batch_size", M3_RENDER_BATCH_SIZE, parse_u64); +/// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16). +static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_compute_batch_size", M3_COMPUTE_BATCH_SIZE, parse_u64); +/// `asahi.m3_early_tiling`: opt-in fragment-only stage dependencies. +static M3_EARLY_TILING: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_early_tiling", M3_EARLY_TILING, parse_u64); + +/// `asahi.m3_early_tiling` is set. +pub(crate) fn early_tiling() -> bool { + M3_EARLY_TILING.load(Ordering::Relaxed) != 0 +} + +fn parse_expose(text: &str) -> Option { + match text.trim() { + "-1" => Some(EXPOSE_AUTO), + other => parse_u64(other).filter(|v| *v <= 1), + } +} +const EXPOSE_AUTO: u64 = u64::MAX; +/// `asahi.m3_expose`: M3 render node registration: -1 = auto (on the boards where the runtime +/// is validated), 0 = never, 1 = always. +static M3_EXPOSE: AtomicU64 = AtomicU64::new(EXPOSE_AUTO); +m3_param!("m3_expose", M3_EXPOSE, parse_expose); + +/// `asahi.m3_expose`: `None` for auto, else whether to register the render node. +pub(crate) fn expose() -> Option { + match M3_EXPOSE.load(Ordering::Relaxed) { + EXPOSE_AUTO => None, + v => Some(v != 0), + } +} + /// Snapshot once per batch. A live change affects later publications, never the /// storage, ordering or retirement checks of work already on the firmware queues. pub(crate) fn render_batch_size() -> usize { let override_size = M3_RENDER_BATCH_OVERRIDE.load(Ordering::Relaxed); let size = if override_size == 0 { - *crate::module_parameters::m3_render_batch_size.value() as usize + M3_RENDER_BATCH_SIZE.load(Ordering::Relaxed) as usize } else { override_size as usize }; - size.clamp(1, crate::m3_pass_layout::SLOTS) + size.clamp(1, crate::m3::m3_pass_layout::SLOTS) } fn parse_compute_batch_override(text: &str) -> Option { - parse_u64(text).filter(|value| *value <= crate::m3_compute_storage::SLOTS as u64) + parse_u64(text).filter(|value| *value <= crate::m3::m3_compute_storage::SLOTS as u64) } /// Independently tune the existing adjacent-compute path. Zero preserves the @@ -229,11 +267,11 @@ m3_param!("m3_compute_batch_override", M3_COMPUTE_BATCH_OVERRIDE, pub(crate) fn compute_batch_size() -> usize { let override_size = M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed); let size = if override_size == 0 { - *crate::module_parameters::m3_compute_batch_size.value() as usize + M3_COMPUTE_BATCH_SIZE.load(Ordering::Relaxed) as usize } else { override_size as usize }; - size.clamp(1, crate::m3_compute_storage::SLOTS) + size.clamp(1, crate::m3::m3_compute_storage::SLOTS) } /// `asahi.g15_debug`: G15 bring-up bits, see [`G15Debug`]. @@ -320,15 +358,6 @@ fn parse_backend(text: &str) -> Option { static M3_BACKEND: AtomicU64 = AtomicU64::new(BACKEND_AUTO); m3_param!("m3_backend", M3_BACKEND, parse_backend); -/// The T6030 GPU backend to start. -pub(crate) enum T6030Backend { - /// The serialized M3 runtime (m3_drm / m3_runtime). - Runtime, - /// The G15 GpuManager backend (g15_probe). - Manager, - /// No backend: the probe stops without touching the GPU. - Off, -} /// `asahi.m3_initdata` values. const INITDATA_AUTO: u64 = 0; @@ -418,7 +447,7 @@ pub(crate) fn unlocked_wait() -> bool { } /// `asahi.m3_ut_engagement=0|1`: Globals +0x7d0, the gate of the firmware's frequency-feedback -/// cap. Unset: the SoC's default ([`crate::m3_soc::Soc::ut_engagement`]). +/// cap. Unset: the SoC's default ([`crate::m3::m3_soc::Soc::ut_engagement`]). static M3_UT_ENGAGEMENT: AtomicU64 = AtomicU64::new(UT_ENGAGEMENT_UNSET); const UT_ENGAGEMENT_UNSET: u64 = u64::MAX; fn parse_ut_engagement(text: &str) -> Option { @@ -427,7 +456,7 @@ fn parse_ut_engagement(text: &str) -> Option { m3_param!("m3_ut_engagement", M3_UT_ENGAGEMENT, parse_ut_engagement); /// The frequency-feedback gate for `soc`, and whether `asahi.m3_ut_engagement` set it. -pub(crate) fn ut_engagement(soc: &crate::m3_soc::Soc) -> (u32, bool) { +pub(crate) fn ut_engagement(soc: &crate::m3::m3_soc::Soc) -> (u32, bool) { match M3_UT_ENGAGEMENT.load(Ordering::Relaxed) { UT_ENGAGEMENT_UNSET => (soc.ut_engagement, false), v => (v as u32, true), @@ -443,14 +472,14 @@ static M3_PIPELINE_DEPTH: AtomicU64 = AtomicU64::new(0); m3_param!("m3_pipeline_depth", M3_PIPELINE_DEPTH, parse_pipeline_depth, 0o644, Some(get_atomic_param)); fn parse_pipeline_depth(text: &str) -> Option { - parse_u64(text).filter(|v| *v<=crate::m3_pass_layout::SLOTS as u64) + parse_u64(text).filter(|v| *v<=crate::m3::m3_pass_layout::SLOTS as u64) } /// Effective depth; changing it never releases already published pass slots. -pub(crate) fn pipeline_depth(soc: &crate::m3_soc::Soc) -> usize { +pub(crate) fn pipeline_depth(soc: &crate::m3::m3_soc::Soc) -> usize { match M3_PIPELINE_DEPTH.load(Ordering::Relaxed) { - 0 => crate::m3_runtime::policy::default_depth(crate::t8122_start::is_t8122(soc)), - v => v.clamp(1,crate::m3_pass_layout::SLOTS as u64) as usize, + 0 => crate::m3::m3_runtime::policy::default_depth(crate::m3::t8122_start::is_t8122(soc)), + v => v.clamp(1,crate::m3::m3_pass_layout::SLOTS as u64) as usize, } } @@ -464,9 +493,9 @@ static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX); m3_param!("m3_render_batch_budget_us", M3_RENDER_BATCH_BUDGET_US, parse_u64, 0o644, Some(get_atomic_param)); /// The render batch GPU-time budget in ns, or `None` without one. -pub(crate) fn render_batch_budget_ns(soc: &crate::m3_soc::Soc) -> Option { +pub(crate) fn render_batch_budget_ns(soc: &crate::m3::m3_soc::Soc) -> Option { let value=M3_RENDER_BATCH_BUDGET_US.load(Ordering::Relaxed); - let us=if value==u64::MAX {crate::m3_runtime::policy::default_budget_us(crate::t8122_start::is_t8122(soc))} else {value}; + let us=if value==u64::MAX {crate::m3::m3_runtime::policy::default_budget_us(crate::m3::t8122_start::is_t8122(soc))} else {value}; match us { 0 => None, us => Some(us.saturating_mul(1000)), @@ -486,12 +515,12 @@ fn parse_retire_mmio(text: &str) -> Option { /// `asahi.m3_retire_mmio`, writable at runtime: the GPU registers the host reads after every /// job before retiring it. Bit 0 reads the engine-busy registers (SGX+0xc020/0xc120), bit 1 /// selects and reads each cluster's fault bank (0xd800/0xd8c0), bit 2 reads the performance -/// state (0xe01000). Unset: the SoC's default ([`crate::m3_soc::Soc::retire_mmio`]). +/// state (0xe01000). Unset: the SoC's default ([`crate::m3::m3_soc::Soc::retire_mmio`]). static M3_RETIRE_MMIO: AtomicU64 = AtomicU64::new(RETIRE_MMIO_UNSET); m3_param!("m3_retire_mmio", M3_RETIRE_MMIO, parse_retire_mmio, 0o644, Some(get_atomic_param)); /// The `asahi.m3_retire_mmio` mask for `soc`. -pub(crate) fn retire_mmio(soc: &crate::m3_soc::Soc) -> u64 { +pub(crate) fn retire_mmio(soc: &crate::m3::m3_soc::Soc) -> u64 { match M3_RETIRE_MMIO.load(Ordering::Relaxed) { RETIRE_MMIO_UNSET => soc.retire_mmio, mask => mask, @@ -563,7 +592,7 @@ pub(crate) fn thermal_hot_param() -> (u64, bool) { /// The runtime starts only on the boards the T8122 table allows (the M3 MacBook Airs), with /// `auto` and `runtime` alike: it has not run on any T8122, so nothing forces it onto another /// board. The G15 manager backend is T6030-only. -pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { +pub(crate) fn t8122_backend(pdev: &platform::Device>) -> bool { let dev = pdev.as_ref(); match M3_BACKEND.load(Ordering::Relaxed) { BACKEND_OFF => { @@ -577,7 +606,7 @@ pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { ); false } - _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T8122) => true, + _ if crate::m3::m3_board::runtime_validated_board(&crate::m3::m3_soc::T8122) => true, _ => { dev_info!( dev, @@ -588,57 +617,27 @@ pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { } } -/// Select the T6030 GPU backend from `asahi.m3_backend`, and say which one and why. -/// -/// `auto` starts the runtime backend on the boards it is validated on (the M3 Pro MacBook -/// Pros). On other T6030 boards it starts nothing until the runtime is validated there; -/// `asahi.m3_backend=runtime` starts it anyway. -pub(crate) fn t6030_backend(pdev: &platform::Device) -> T6030Backend { - let dev = pdev.as_ref(); - match M3_BACKEND.load(Ordering::Relaxed) { - BACKEND_RUNTIME => { - dev_info!(dev, "M3: asahi.m3_backend=runtime\n"); - T6030Backend::Runtime - } - BACKEND_MANAGER => { - dev_info!(dev, "M3: asahi.m3_backend=manager: starting the G15 manager backend\n"); - T6030Backend::Manager - } - BACKEND_OFF => { - dev_info!(dev, "M3: asahi.m3_backend=off: no GPU backend started\n"); - T6030Backend::Off - } - _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T6030) => T6030Backend::Runtime, - _ => { - dev_info!( - dev, - "M3: asahi.m3_backend=auto: no GPU backend on this board: the runtime backend is not validated here yet (asahi.m3_backend=runtime starts it, asahi.m3_backend=manager starts the G15 manager backend)\n" - ); - T6030Backend::Off - } - } -} fn parse_t8122_number(text: &str) -> Option { - Some(crate::t8122_knobs::parse_number(text)) + Some(crate::m3::t8122_knobs::parse_number(text)) } fn parse_t8122_fender(text: &str) -> Option { - Some(crate::t8122_knobs::parse_fender(text)) + Some(crate::m3::t8122_knobs::parse_fender(text)) } fn parse_t8122_clkgen(text: &str) -> Option { - Some(crate::t8122_knobs::parse_clkgen(text)) + Some(crate::m3::t8122_knobs::parse_clkgen(text)) } fn parse_t8122_sgx_setup(text: &str) -> Option { - Some(crate::t8122_knobs::parse_sgx_setup(text)) + Some(crate::m3::t8122_knobs::parse_sgx_setup(text)) } // The T8122 start experiment (`t8122_start`). Each parameter is stored as given, unset or with a // value its parser does not accept (`t8122_knobs`), so that the experiment can refuse rather // than fall back to a default. -use crate::t8122_knobs::UNSET as T8122_UNSET; +use crate::m3::t8122_knobs::UNSET as T8122_UNSET; /// `asahi.t8122_start=1`: start the M3 runtime on a T8122 with the start experiment's values. /// Only with a boot loader that handed the GPU over. @@ -667,21 +666,21 @@ static T8122_PSTATE_CAP: AtomicU64 = AtomicU64::new(T8122_UNSET); m3_param!("t8122_pstate_cap", T8122_PSTATE_CAP, parse_t8122_number); /// `asahi.t8122_hwdata_418c=0|1`: on a T8122 with the G15 14.8.3 firmware, 1 (the default) -/// writes the 20 words at HwDataA +0x418c ([`crate::t8122_hwdata`]); 0 leaves the +/// writes the 20 words at HwDataA +0x418c ([`crate::m3::t8122_hwdata`]); 0 leaves the /// range zero, for an A/B test. Readable in sysfs. -static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::t8122_hwdata::DEFAULT); -m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::t8122_hwdata::parse, 0o444, +static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::m3::t8122_hwdata::DEFAULT); +m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::m3::t8122_hwdata::parse, 0o444, Some(get_atomic_param)); /// Whether `asahi.t8122_hwdata_418c` asks for the HwDataA +0x418c words (the default). pub(crate) fn t8122_hwdata_418c() -> bool { - crate::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) + crate::m3::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) } /// The T8122 start-experiment parameters of this boot, as given. -pub(crate) fn t8122_params() -> crate::t8122_knobs::Raw { +pub(crate) fn t8122_params() -> crate::m3::t8122_knobs::Raw { let get = |p: &AtomicU64| p.load(Ordering::Relaxed); - crate::t8122_knobs::Raw { + crate::m3::t8122_knobs::Raw { start: get(&T8122_START), initdata_version: get(&T8122_INITDATA_VERSION), fender: get(&T8122_FENDER), @@ -692,209 +691,3 @@ pub(crate) fn t8122_params() -> crate::t8122_knobs::Raw { pstate_cap: get(&T8122_PSTATE_CAP), } } - -fn parse_t6031_number(text: &str) -> Option { - Some(crate::t6031_knobs::parse_number(text)) -} - -fn parse_t6031_fender(text: &str) -> Option { - Some(crate::t6031_knobs::parse_fender(text)) -} - -fn parse_t6031_clkgen(text: &str) -> Option { - Some(crate::t6031_knobs::parse_clkgen(text)) -} - -fn parse_t6031_sgx_setup(text: &str) -> Option { - Some(crate::t6031_knobs::parse_sgx_setup(text)) -} - -fn parse_t6031_mcache(text: &str) -> Option { - Some(crate::t6031_knobs::parse_mcache(text)) -} - -fn parse_t6031_aic_swint(text: &str) -> Option { - Some(crate::t6031_knobs::parse_aic_swint(text)) -} - -fn parse_t6031_slot7(text: &str) -> Option { - Some(crate::t6031_knobs::parse_slot7(text)) -} - -fn parse_t6031_gifaf(text: &str) -> Option { - Some(crate::t6031_knobs::parse_gifaf(text)) -} - -fn parse_t6031_slot(text: &str) -> Option { - Some(crate::t6031_knobs::parse_slot(text)) -} - -fn parse_t6031_ane(text: &str) -> Option { - Some(crate::t6031_knobs::parse_ane(text)) -} - -fn parse_t6031_mtr(text: &str) -> Option { - Some(crate::t6031_knobs::parse_mtr(text)) -} - -fn parse_t6031_fw_words(text: &str) -> Option { - Some(crate::t6031_knobs::parse_fw_words(text)) -} - -fn parse_t6031_hwdata_object(text: &str) -> Option { - Some(crate::t6031_knobs::parse_hwdata_object(text)) -} - -fn parse_t6031_rev_id(text: &str) -> Option { - Some(crate::t6031_knobs::parse_rev_id(text)) -} - -fn parse_t6031_tristate(text: &str) -> Option { - Some(crate::t6031_knobs::parse_tristate(text)) -} - -fn parse_t6031_tag(text: &str) -> Option { - Some(crate::t6031_knobs::parse_tag(text)) -} - -// The T6031 start (`t6031_start`). Each parameter is stored as given, unset or with a value its -// parser does not accept (`t6031_knobs`), so the start can refuse rather than fall back. -use crate::t6031_knobs::UNSET as T6031_UNSET; - -/// `asahi.t6031_start=1`: one attempt to start the M3 runtime on a T6031. Without it the probe -/// is the same refusal as before. -static T6031_START: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_start", T6031_START, parse_t6031_number); -/// `asahi.t6031_image_hash`: the first 8 bytes of the loaded text SHA-256, big-endian. -static T6031_IMAGE_HASH: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_image_hash", T6031_IMAGE_HASH, parse_t6031_number); -/// `asahi.t6031_tag=search` or a text offset of the boot-entropy tag. -static T6031_TAG: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_tag", T6031_TAG, parse_t6031_tag); -/// `asahi.t6031_initdata_version=`: the InitData version given to the T6031 firmware. -static T6031_INITDATA_VERSION: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_initdata_version", T6031_INITDATA_VERSION, parse_t6031_number); -/// `asahi.t6031_pstate_cap=N`: the highest performance state the T6031 firmware may use. -static T6031_PSTATE_CAP: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_pstate_cap", T6031_PSTATE_CAP, parse_t6031_number); -/// `asahi.t6031_fender=rule|adt`: the Fender window size. -static T6031_FENDER: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_fender", T6031_FENDER, parse_t6031_fender); -/// `asahi.t6031_clkgen=e5c|e1c|none`: the GPU clock-generator IO mapping. -static T6031_CLKGEN: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_clkgen", T6031_CLKGEN, parse_t6031_clkgen); -/// `asahi.t6031_sgx_setup=t6030|none`: the SGX write made before the firmware starts. -static T6031_SGX_SETUP: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_sgx_setup", T6031_SGX_SETUP, parse_t6031_sgx_setup); -/// `asahi.t6031_mcache=none|x2|x4|x8`. -static T6031_MCACHE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mcache", T6031_MCACHE, parse_t6031_mcache); -/// `asahi.t6031_aic_swint=page|reg`. -static T6031_AIC_SWINT: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_aic_swint", T6031_AIC_SWINT, parse_t6031_aic_swint); -/// `asahi.t6031_slot7=none|rule`. -static T6031_SLOT7: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot7", T6031_SLOT7, parse_t6031_slot7); -/// `asahi.t6031_gifaf=none|fender`. -static T6031_GIFAF: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_gifaf", T6031_GIFAF, parse_t6031_gifaf); -/// `asahi.t6031_slot21=none` or a physical address. -static T6031_SLOT21: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot21", T6031_SLOT21, parse_t6031_slot); -/// `asahi.t6031_slot24=none` or a physical address. -static T6031_SLOT24: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot24", T6031_SLOT24, parse_t6031_slot); -/// `asahi.t6031_slot28=none` or a physical address. -static T6031_SLOT28: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot28", T6031_SLOT28, parse_t6031_slot); -/// `asahi.t6031_ane=none|t6030|t8122` or a physical address. -static T6031_ANE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_ane", T6031_ANE, parse_t6031_ane); -/// `asahi.t6031_io_drop`: a mask of default slots to leave out. -static T6031_IO_DROP: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_io_drop", T6031_IO_DROP, parse_t6031_number); -/// `asahi.t6031_mtr=t6030|t8122|adt|none`. -static T6031_MTR: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mtr", T6031_MTR, parse_t6031_mtr); -/// `asahi.t6031_mtr_fast_die`: overrides the fast-die MTR mask. -static T6031_MTR_FAST_DIE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mtr_fast_die", T6031_MTR_FAST_DIE, parse_t6031_number); -/// `asahi.t6031_mtr_alarm`: overrides the alarm MTR mask. -static T6031_MTR_ALARM: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mtr_alarm", T6031_MTR_ALARM, parse_t6031_number); -/// `asahi.t6031_unit_mask_a=`: HwDataB unit mask A (+0x17c0). -static T6031_UNIT_MASK_A: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_unit_mask_a", T6031_UNIT_MASK_A, parse_t6031_number); -/// `asahi.t6031_unit_mask_b=`: HwDataB unit mask B (+0x17c8). -static T6031_UNIT_MASK_B: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_unit_mask_b", T6031_UNIT_MASK_B, parse_t6031_number); -/// `asahi.t6031_hwb_454`: HwDataB +0xa2c. -static T6031_HWB_454: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_454", T6031_HWB_454, parse_t6031_number); -/// `asahi.t6031_hwb_b20`: HwDataB +0xb20, the core slots. -static T6031_HWB_B20: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_b20", T6031_HWB_B20, parse_t6031_number); -/// `asahi.t6031_hwb_17b8`: HwDataB +0x17b8. -static T6031_HWB_17B8: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_17b8", T6031_HWB_17B8, parse_t6031_number); -/// `asahi.t6031_hwb_1818`: HwDataB +0x1818. -static T6031_HWB_1818: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_1818", T6031_HWB_1818, parse_t6031_number); -/// `asahi.t6031_fw_words=none|t8122`. -static T6031_FW_WORDS: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_fw_words", T6031_FW_WORDS, parse_t6031_fw_words); -/// `asahi.t6031_hwdata_object=aligned|fixed`. -static T6031_HWDATA_OBJECT: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwdata_object", T6031_HWDATA_OBJECT, parse_t6031_hwdata_object); -/// `asahi.t6031_gpu_core`: must be the table's G15C core type, 24. -static T6031_GPU_CORE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_gpu_core", T6031_GPU_CORE, parse_t6031_number); -/// `asahi.t6031_rev_id=auto` or 1 to 6. -static T6031_REV_ID: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_rev_id", T6031_REV_ID, parse_t6031_rev_id); -/// `asahi.t6031_csafr=auto|on|off`. -static T6031_CSAFR: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_csafr", T6031_CSAFR, parse_t6031_tristate); -/// `asahi.t6031_power_cap_mw`: the power target while the boot loader's model is a stand-in. -static T6031_POWER_CAP_MW: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_power_cap_mw", T6031_POWER_CAP_MW, parse_t6031_number); - -/// The T6031 start parameters of this boot, as given. The image UUID has no module parameter. -pub(crate) fn t6031_params() -> crate::t6031_knobs::Raw { - let get = |p: &AtomicU64| p.load(Ordering::Relaxed); - crate::t6031_knobs::Raw { - start: get(&T6031_START), - image_uuid: crate::t6031_knobs::UuidParam::Unset, - image_hash: get(&T6031_IMAGE_HASH), - tag: get(&T6031_TAG), - initdata_version: get(&T6031_INITDATA_VERSION), - pstate_cap: get(&T6031_PSTATE_CAP), - fender: get(&T6031_FENDER), - clkgen: get(&T6031_CLKGEN), - sgx_setup: get(&T6031_SGX_SETUP), - mcache: get(&T6031_MCACHE), - aic_swint: get(&T6031_AIC_SWINT), - slot7: get(&T6031_SLOT7), - gifaf: get(&T6031_GIFAF), - slot21: get(&T6031_SLOT21), - slot24: get(&T6031_SLOT24), - slot28: get(&T6031_SLOT28), - ane: get(&T6031_ANE), - io_drop: get(&T6031_IO_DROP), - mtr: get(&T6031_MTR), - mtr_fast_die: get(&T6031_MTR_FAST_DIE), - mtr_alarm: get(&T6031_MTR_ALARM), - unit_mask_a: get(&T6031_UNIT_MASK_A), - unit_mask_b: get(&T6031_UNIT_MASK_B), - hwb_454: get(&T6031_HWB_454), - hwb_b20: get(&T6031_HWB_B20), - hwb_17b8: get(&T6031_HWB_17B8), - hwb_1818: get(&T6031_HWB_1818), - fw_words: get(&T6031_FW_WORDS), - hwdata_object: get(&T6031_HWDATA_OBJECT), - gpu_core: get(&T6031_GPU_CORE), - rev_id: get(&T6031_REV_ID), - csafr: get(&T6031_CSAFR), - power_cap_mw: get(&T6031_POWER_CAP_MW), - } -} diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs index 273d2701c3f14..94c2172e33279 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs @@ -2,12 +2,12 @@ //! Private storage for one retired/unpublished G15 render pass. A Pass retains //! all its buffers and mappings until Render retires the shared queues. use kernel::prelude::*; -use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_pass_layout as layout}; +use crate::{driver,mmu,pgtable::prot,m3::m3_memory::Buffer,m3::m3_pass_layout as layout}; use layout::{Field,Space}; pub(crate) struct Pass {buffers:KVec} impl Pass { - pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, + pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm, slot:usize,clusters:u32,aliases:&mut KVec)->Result { let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?; for field in layout::FIELDS { @@ -28,7 +28,7 @@ impl Pass { pub(crate) fn get_mut(&mut self,field:Field)->&mut Buffer {&mut self.buffers[field as usize]} /// Caller must prove retirement, and must not clear a prepared sibling. /// Dependency commands are rewritten completely by the sync encoder. - pub(crate) fn reset(&mut self,geometry:&crate::agx_render::Geometry)->Result { + pub(crate) fn reset(&mut self,geometry:&crate::m3::agx_render::Geometry)->Result { // Tiler arenas grow to retain the largest prior target. The firmware // descriptors use the current geometry's strides and layer count, so // only that prefix is addressable by this pass. Clear it on every use, diff --git a/drivers/gpu/drm/asahi/m3/m3_pool.rs b/drivers/gpu/drm/asahi/m3/m3_pool.rs index 979c1a37bb202..ab50ab7eca8a1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pool.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pool.rs @@ -3,7 +3,7 @@ //! GPU mappings and counters outlive this initialization and remain owned by //! Render/Compute. Callers must never invoke this on a live or replayed pool. use kernel::prelude::*; -use crate::{m3_memory::Buffer, m3_pool_layout as wire, m3_queue_layout::FirmwareVa}; +use crate::{m3::m3_memory::Buffer, m3::m3_pool_layout as wire, m3::m3_queue_layout::FirmwareVa}; pub(crate) struct Owners { pub(crate) backing: core::ops::Range, pub(crate) pages: usize, diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs index c2c67f5c3aa5a..e7a02982e96f1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT use kernel::prelude::*; -use crate::m3_queue_layout as queue; -use crate::m3_sync_layout as sync; -use crate::m3_state_layout as state; -use crate::m3_scene_layout as scene; -use crate::{m3_pass::{Pass,ViewOwner},m3_pass_layout::{self as pass_layout,Field as P,Space}}; -use crate::{m3_fragment_command as fragment, m3_tiler_command as tiler, m3_render_sequence as sequence, m3_init_layout::Region}; -use crate::{m3_parameter_layout as parameter, m3_pool_layout::GpuRegion}; -use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_shared_layout as s,agx_uapi}; -use crate::m3_timeline as timeline; +use crate::m3::m3_queue_layout as queue; +use crate::m3::m3_sync_layout as sync; +use crate::m3::m3_state_layout as state; +use crate::m3::m3_scene_layout as scene; +use crate::{m3::m3_pass::{Pass,ViewOwner},m3::m3_pass_layout::{self as pass_layout,Field as P,Space}}; +use crate::{m3::m3_fragment_command as fragment, m3::m3_tiler_command as tiler, m3::m3_render_sequence as sequence, m3::m3_init_layout::Region}; +use crate::{m3::m3_parameter_layout as parameter, m3::m3_pool_layout::GpuRegion}; +use crate::{driver,mmu,pgtable::prot,m3::m3_memory::Buffer,m3::m3_shared_layout as s,m3::agx_uapi}; +use crate::m3::m3_timeline as timeline; use s::{PB_FIRST,PB_GROUPS,PB_BLOCKS_PER_GROUP,PB_BLOCK_SIZE}; const PB_BLOCKS:u32=(PB_GROUPS*PB_BLOCKS_PER_GROUP) as u32; pub(crate) const PB_PAGES:u32=PB_BLOCKS*4; @@ -38,7 +38,9 @@ pub(crate) struct Render { /// GPU clusters of the board: tiler accelerators (TPC storage) and preemption buffers. clusters:u32, /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). - registers:crate::m3_compute_layout::RegisterSet, + registers:crate::m3::m3_compute_layout::RegisterSet, + /// Last: drops after the pass aliases. + _lower_tlbi:crate::m3::m3_memory::LowerTlbi, } // Queue allocations belong to Render, not to a pass. Keep their fixed arenas // and firmware/shared mapping permissions from the qualified allocation map. @@ -51,9 +53,9 @@ use s::{GPU_CONTEXT,JOB_LIST,BUFFER_MANAGER,PARAMETER_PAGE_LIST, PARAMETER_BLOCK_LIST,NOTIFIER,BLOCK_CONTROL,MANAGER_MISC}; const MANAGER_SCENE_COUNT: u32 = 48; const TA_POOL_COUNTER: usize = 11; -const UMA_POOLS: [crate::m3_pool::Owners; 2] = [ - crate::m3_pool::Owners { backing: 0..8, pages: 8, table: 9, manager: 10, counter: 11, table_index: 0 }, - crate::m3_pool::Owners { backing: 12..20, pages: 20, table: 21, manager: 22, counter: 23, table_index: 1 }, +const UMA_POOLS: [crate::m3::m3_pool::Owners; 2] = [ + crate::m3::m3_pool::Owners { backing: 0..8, pages: 8, table: 9, manager: 10, counter: 11, table_index: 0 }, + crate::m3::m3_pool::Owners { backing: 12..20, pages: 20, table: 21, manager: 22, counter: 23, table_index: 1 }, ]; const FRAGMENT_POOL_COUNTER: usize = 23; // Retained producer identifier for cross-pass dependencies, G15 / V14_8_3. @@ -65,8 +67,9 @@ const RENDER_QUEUES: [QueueObjects; 2] = [ scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 }, ]; impl Render { - pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, - registers:crate::m3_compute_layout::RegisterSet)->Result { + pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, + registers:crate::m3::m3_compute_layout::RegisterSet)->Result { + let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new();let mut aliases=KVec::new(); for index in 0..s::COUNT { @@ -91,14 +94,14 @@ impl Render { tiler_bytes.resize(tiler::SIZE,0u8,GFP_KERNEL)?; let mut fragment_bytes=KVec::new(); fragment_bytes.resize(fragment::SIZE,0u8,GFP_KERNEL)?; - let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers}; + let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; // Lists and the matching manager are built before InitBM publication. job.initialize_parameter_buffer()?; for pool in &UMA_POOLS { pool.initialize(&mut job.objects)?; } job.initialize_state()?; job.initialize_queues()?; dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024)); - crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + crate::m3::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; Ok(job) } @@ -223,7 +226,7 @@ impl Render { // Prepare each private command after its retired slot is reset. The // reusable host staging allocation is copied before the next slot is built. fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,draw:u64)->Result { - use crate::m3_compute_layout::GpuVa; + use crate::m3::m3_compute_layout::GpuVa; let pass=&self.passes[self.slot]; let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); @@ -247,7 +250,7 @@ impl Render { user_timestamps:[fw(P::TilerUserStart)?,fw(P::TilerUserEnd)?],preemption:[gpu(P::Preemption0)?,gpu(P::Preemption1)?,gpu(P::Preemption2)?], tilemap:gpu(P::Tilemap)?,tpc:gpu(P::Tpc)?,tpc_bytes:g.tpc_bytes,heap:gpu(P::HeapMetadata)?, scene_user:gpu(P::SceneUser)?,initial_scene_entry,geometry:g,page_count:PB_PAGES, - vdm:crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, + vdm:crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0, }; value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?; @@ -290,8 +293,8 @@ impl Render { Ok(()) } /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster. - fn geometry(r:agx_uapi::UapiRenderCommand,clusters:u32)->Result { - use crate::agx_render::{Geometry,Utile}; + fn geometry(r:agx_uapi::UapiRenderCommand,clusters:u32)->Result { + use crate::m3::agx_render::{Geometry,Utile}; let x=if r.utile_width==16 {Utile::Pixels16} else {Utile::Pixels32}; let y=if r.utile_height==16 {Utile::Pixels16} else {Utile::Pixels32}; let mut g=Geometry::new_layered(u32::from(r.width),u32::from(r.height),x,y,clusters,r.layers).map_err(|_|EINVAL)?; @@ -299,7 +302,7 @@ impl Render { Ok(g) } fn prepare_fragment(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result { - use crate::{m3_compute_layout::GpuVa,agx_render_state::{Program,DepthStencil}}; + use crate::{m3::m3_compute_layout::GpuVa,m3::agx_render_state::{Program,DepthStencil}}; let pass=&self.passes[self.slot]; let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); @@ -355,7 +358,7 @@ impl Render { self.passes[self.slot].get_mut(sequence).u64(start+sequence::USER_TIMESTAMP_POINTER,pointer)?; self.passes[self.slot].get_mut(sequence).u64(end+sequence::USER_TIMESTAMP_POINTER,pointer)?; } - crate::agx_memory::publish();Ok(()) + crate::m3::agx_memory::publish();Ok(()) } fn at(base:usize,index:usize)->usize {(base+index)%pass_layout::SLOTS} /// GPU time of the batch of `count` passes starting at pass slot `base`. @@ -416,13 +419,13 @@ impl Render { /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then /// the VM must already be bound, no storage may grow and the slots must not be in flight /// (the caller keeps the in-flight passes plus this batch within SLOTS). - pub(crate) fn begin_batch(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, - commands:&[crate::m3_submit::Command],base:usize,overlap:bool)->Result { + pub(crate) fn begin_batch(&mut self,dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm, + commands:&[crate::m3::m3_submit::Command],base:usize,overlap:bool)->Result { if base>=pass_layout::SLOTS || (!overlap && !self.complete()?) {return Err(EBUSY);} if overlap { if !self._binding.matches(vm) {return Err(EBUSY);} for (index,&command) in commands.iter().enumerate() { - let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; let g=Self::geometry(r,self.clusters)?; let slot=Self::at(base,index); if g.tpc_bytes>self.passes[slot].get(P::Tpc).size() as u64 @@ -433,9 +436,9 @@ impl Render { self.draw.checked_add(commands.len() as u64).ok_or(EOVERFLOW)?; // Validate every pass before changing any queue or completion state. for &command in commands { - let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; Self::geometry(r,self.clusters)?; - crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?; + crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?; } if !self._binding.matches(vm) { let next=if let Some(index)=self.cached_views.iter().position(|v|v.binding.matches(vm)) { @@ -477,7 +480,7 @@ impl Render { // already-grown buffers are reusable on the next submission. for (index,&command) in commands.iter().enumerate() { let slot=Self::at(base,index); - let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; let g=Self::geometry(r,self.clusters)?; for (field,base,bytes) in [(P::Tpc,s::TPC_LABEL,g.tpc_bytes),(P::Tilemap,s::TILEMAP_LABEL,g.tilemap_bytes)] { let size=usize::try_from(bytes).map_err(|_|EOVERFLOW)?; @@ -504,7 +507,7 @@ impl Render { for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() { self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; } - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); Ok(()) } /// Append only into caller-reserved free pass slots. With overlap, WRITE updates @@ -533,7 +536,7 @@ impl Render { // Retain the conservative stage-scope permission. Only a pass after the // first of its batch may overlap its predecessor's stages. Explicit // start timestamps keep the original full-order execution boundary. - let early=predecessor.is_none() && self.batch_count>0 && *crate::module_parameters::m3_early_tiling.value()!=0 + let early=predecessor.is_none() && self.batch_count>0 && crate::m3::m3_params::early_tiling() && r.flags&(1<<5)!=0 && r.vertex_timestamps.start.handle==0 && r.fragment_timestamps.start.handle==0; if early {self.early_count+=1;} @@ -585,15 +588,15 @@ impl Render { self.objects[ring].u64(usize::from(self.heads[stage])*8,address)?; self.heads[stage]=(self.heads[stage]+1)%0x500; } - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; } - crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();Ok(()) + crate::m3::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();Ok(()) } /// Emitted only after both events, stamp/queue checks, clear faults, /// idle engines and consumed firmware pipes have been checked by Runtime. /// The existing TA/fragment descriptors request completion stamp flushes. - pub(crate) fn progress(&mut self,dev:&driver::AsahiDevice)->Result { + pub(crate) fn progress(&mut self,dev:&driver::AsahiDrmDevice)->Result { // Keep summaries sparse at high throughput, but emit at least once a // second while completed batches advance. Batched draw counts can skip // multiples of 128; a count-only gate can starve the watchdog observer. @@ -654,7 +657,7 @@ impl Render { || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} } Ok(true) } - pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { // Retain allocator state as well as payloads: an exhausted parameter // pool can stall TA without producing an MMU fault. for (index,bytes) in [(BLOCK_CONTROL,64),(s::BM_COUNTER,64),(MANAGER_MISC,64),(BUFFER_MANAGER,172)] { diff --git a/drivers/gpu/drm/asahi/m3/m3_resources.rs b/drivers/gpu/drm/asahi/m3/m3_resources.rs index bfc94fdfa0ca8..7d34305250018 100644 --- a/drivers/gpu/drm/asahi/m3/m3_resources.rs +++ b/drivers/gpu/drm/asahi/m3/m3_resources.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! M3 GPU resources admitted against the measured boot contract (J514S on T6030). -pub(crate) use crate::agx_resources::{Region, Resources}; +pub(crate) use crate::m3::agx_resources::{Region, Resources}; use kernel::{bindings, c_str, io::resource::Resource, of, prelude::*}; @@ -11,13 +11,13 @@ pub(crate) fn reserved_resource(node: &of::Node, name: &CStr) -> Result = node.get_property(c_str!("compatible"))?; let compatible = compatible.strip_suffix(b"\0").ok_or(EINVAL)?; - if !crate::m3_soc::SOCS.iter().any(|soc| compatible == soc.gpu.as_bytes()) { + if !crate::m3::m3_soc::SOCS.iter().any(|soc| compatible == soc.gpu.as_bytes()) { return Err(EINVAL); } let names: KVec = node.get_property(c_str!("memory-region-names"))?; @@ -44,8 +44,8 @@ pub(crate) fn reserved_resource(node: &of::Node, name: &CStr) -> Result, - soc: &crate::m3_soc::Soc, + pdev: &kernel::platform::Device>, + soc: &crate::m3::m3_soc::Soc, ) -> kernel::error::Result { - crate::m3_board::admit(pdev, soc) + crate::m3::m3_board::admit(pdev, soc) } diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs index 3e8cbbfa93781..be81c4a4e9d92 100644 --- a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -3,18 +3,17 @@ //! RTKit callbacks for the identified J514S firmware. The firmware advertises //! its crash storage by physical address inside the reserved data segment. -use crate::agx_resources::Region; +use crate::m3::agx_resources::Region; use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use kernel::{ c_str, impl_has_hr_timer, device::Core, sync::Completion, - io::mem::{Mem, MemFlag}, - iosys_map::IoSysMapRef, + io::{mem::{Mem, MemFlag}, IoSysMap, Region as IoRegion, SysMem}, platform, prelude::*, soc::apple::rtkit, - sync::{aref::ARef, Arc, ArcBorrow}, + sync::{aref::ARef, Arc, ArcBorrow, SetOnce}, time::{Delta, Monotonic, hrtimer::{HrTimer, HrTimerCallback, HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode}}, }; @@ -40,7 +39,7 @@ impl EventWait { pin_init!(EventWait { count: AtomicU64::new(0), wait_us: AtomicU64::new(100), - overlap_cpu: AtomicBool::new(crate::m3_params::unlocked_wait()), + overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()), cpu_preparations: AtomicU64::new(0), sleeps: AtomicU64::new(0), timer_wakes: AtomicU64::new(0), @@ -139,6 +138,9 @@ pub(crate) struct State { /// The firmware sends one after a command's completion processing. pub(crate) event_messages: AtomicU64, pub(crate) events: Arc, + /// The completion worker, published once the runtime it services exists. The DRM device + /// carries no driver data for it on 7.2, so the runtime's own state owns it. + pub(crate) completion: SetOnce>, } /// Shared status contains no device references, so DRM files may retain it @@ -147,8 +149,8 @@ pub(crate) struct Health { crashed: AtomicBool, mapped: AtomicBool, failed: AtomicBool, - progress: crate::agx_host_progress::Progress, - pub(crate) timing: crate::agx_timing_stats::Stats, + progress: crate::m3::agx_host_progress::Progress, + pub(crate) timing: crate::m3::agx_timing_stats::Stats, } impl Health { @@ -181,7 +183,7 @@ impl Health { } impl State { - pub(crate) fn new(dev: &platform::Device, drm: crate::driver::AsahiDevRef, data: Region) -> Result> { + pub(crate) fn new(dev: &platform::Device>, drm: crate::driver::AsahiDevRef, data: Region) -> Result> { Ok(Arc::new( Self { dev: dev.into(), @@ -191,12 +193,13 @@ impl State { crashed: AtomicBool::new(false), mapped: AtomicBool::new(false), failed: AtomicBool::new(false), - progress: crate::agx_host_progress::Progress::new(), - timing: crate::agx_timing_stats::Stats::new(), + progress: crate::m3::agx_host_progress::Progress::new(), + timing: crate::m3::agx_timing_stats::Stats::new(), }, GFP_KERNEL)?, claimed: AtomicBool::new(false), event_messages: AtomicU64::new(0), events: EventWait::new()?, + completion: SetOnce::new(), }, GFP_KERNEL, )?) @@ -227,8 +230,18 @@ impl rtkit::Buffer for FirmwareBuffer { fn iova(&self) -> Result { Ok(self.physical) } - fn buf(&mut self) -> Result> { - self.mapping.iosys_map(self.offset, self.size) + fn buf(&mut self) -> Result> { + // The view of 7.1.12's `Mem::iosys_map`: the bounded span inside the retained WB + // mapping, as normal memory. + if self.offset.checked_add(self.size).ok_or(EOVERFLOW)? > self.mapping.size() { + return Err(EINVAL); + } + // SAFETY: The span lies entirely inside the retained mapping (checked above). + let base = unsafe { self.mapping.ptr().add(self.offset) }; + let region = IoRegion::ptr_try_from_raw_parts_mut(base, self.size)?; + // SAFETY: The span is mapped kernel memory for as long as `self.mapping`, which the + // returned view borrows through `&mut self`. + Ok(IoSysMap::Sys(unsafe { SysMem::new(region) })) } } @@ -244,15 +257,12 @@ impl rtkit::Operations for Operations { // Wake the serialized worker even if the firmware can no longer send // its usual completion notification. Never take the runtime lock here. state.events.record(); - crate::m3_completion::queue(&state.drm); + crate::m3::m3_completion::queue(&state); dev_err!( state.dev.as_ref(), "M3 G15S: firmware crashed, retained crashlog bytes={}\n", crashlog.map_or(0, |b| b.len()) ); - if let Some(bytes) = crashlog { - crate::t6031_start::capture_crashlog(state.dev.as_ref(), bytes); - } } fn shmem_map( @@ -275,7 +285,7 @@ impl rtkit::Operations for Operations { return Err(EBUSY); } let node = state.dev.as_ref().of_node().ok_or(ENODEV)?; - let resource = crate::m3_resources::reserved_resource(&node,c_str!("fw-data"))?; + let resource = crate::m3::m3_resources::reserved_resource(&node,c_str!("fw-data"))?; if resource.start() != state.data.base || resource.size() != state.data.size { return Err(EINVAL); } @@ -314,7 +324,7 @@ impl rtkit::Operations for Operations { ::ktime_get(), state.event_messages.load(Ordering::Acquire)); } - crate::m3_completion::queue(&state.drm); + crate::m3::m3_completion::queue(&state); return; } dev_warn!( diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 08885f591c140..b12fc73f9f248 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -2,19 +2,19 @@ use core::{mem::ManuallyDrop, sync::atomic::Ordering}; use kernel::{device::Core, platform, prelude::*, soc::apple::rtkit, sync::Arc, time::{Instant, Monotonic, Delta, delay::fsleep}}; -use crate::{driver, mmu, m3_device::Device, m3_config::Config, m3_rtkit}; +use crate::{driver, mmu, m3::m3_device::Device, m3::m3_config::Config, m3::m3_rtkit}; #[path = "m3_batch_policy.rs"] pub(crate) mod policy; -const _: () = assert!(policy::SLOTS == crate::m3_pass_layout::SLOTS); +const _: () = assert!(policy::SLOTS == crate::m3::m3_pass_layout::SLOTS); -enum NativeJob {Compute(crate::m3_compute::Compute),Render(crate::m3_render::Render)} +enum NativeJob {Compute(crate::m3::m3_compute::Compute),Render(crate::m3::m3_render::Render)} /// Completion-worker outcome. pub(crate) enum Service {Idle,Progress,Waiting(u64)} /// The published batch: its commands in order, as (packet, command index). struct Batch { - entries: KVec<(Arc,usize)>, - index: usize, kind: usize, first: crate::m3_submit::Command, + entries: KVec<(Arc,usize)>, + index: usize, kind: usize, first: crate::m3::m3_submit::Command, preparation_ns: i64, previous_events: u64, expected_events: u64, start: Instant, measure: bool, doorbell_counter_ns: i64, /// Render: first pass slot and the batch's last draw number. @@ -40,16 +40,16 @@ struct GeometryTiming { } impl NativeJob { fn complete(&mut self)->Result {match self {Self::Compute(j)=>j.complete(),Self::Render(j)=>j.complete()}} - fn log(&mut self,dev:&driver::AsahiDevice)->Result {match self {Self::Compute(j)=>j.log(dev),Self::Render(j)=>j.log(dev)}} + fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result {match self {Self::Compute(j)=>j.log(dev),Self::Render(j)=>j.log(dev)}} fn batch_gpu_span(&mut self)->Result<[u64;2]> {match self {Self::Compute(j)=>j.batch_gpu_span(),Self::Render(j)=>j.batch_gpu_span()}} } struct Inner { transport: rtkit::RtKit, state: Arc, jobs: KVec, - packets: KVec>, + packets: KVec>, /// Packets not yet (fully) published: (packet, next command index), most urgent first. - pending: KVec<(Arc,usize)>, + pending: KVec<(Arc,usize)>, /// Published batches, oldest first. More than one only with asahi.m3_pipeline_depth > 1. active: KVec, gpu_pending: bool, @@ -57,14 +57,14 @@ struct Inner { active_power_samples: u32, fault_captured: bool, timing: [[i64;7];2], - render_batches: [u64;crate::m3_pass_layout::SLOTS], + render_batches: [u64;crate::m3::m3_pass_layout::SLOTS], geometry: [GeometryTiming; 32], geometry_overflow: u64, /// Whether a T8122 job verdict line was already emitted for the current job, so the /// execute-path fallback does not add a second one. T8122 only. t8122_verdict: bool, /// The step the firmware boot sequence has reached, for the T8122 boot verdict. T8122 only. - boot_step: crate::t8122_start::BootStep, + boot_step: crate::m3::t8122_start::BootStep, /// Packets that joined a batch started by an earlier packet (debug summary). coalesced_packets: u64, coalesced_render_stamps: u64, @@ -76,6 +76,9 @@ struct Inner { render_initialized: bool, config: Config, uat: mmu::Uat, + /// The firmware's view of the host window (the shared L2 the firmware walks), kept in step + /// with the kernel VM's shadow window before the firmware is given new kernel addresses. + window: crate::g15::uat::Window, drm: driver::AsahiDevRef, device: Device, } @@ -86,10 +89,10 @@ impl Inner { self.state.health.mark_failed(); if self.fault_captured { return; } self.fault_captured = true; - #[cfg(CONFIG_DEV_COREDUMP)] - if let Err(error) = self.config.fault_snapshot(&self.drm, primary, self.gpu_pending) { - dev_err!(self.drm.as_ref(), "M3 fault snapshot failed: {:?}\n", error); - } + // 7.1.12 also published a devcoredump of the firmware objects here; 7.2's kernel crate + // has no devcoredump binding, so the first error is only logged. + dev_err!(self.drm.as_ref(), "M3: first GPU failure: {:?} (GPU work pending: {})\n", + primary, self.gpu_pending); } /// Record a batch that failed or never retired. Mark the GPU failed first, @@ -97,15 +100,12 @@ impl Inner { /// behind a device that still reports itself healthy. fn fail(&mut self, index: usize, vm: &mmu::Vm, primary: Error) { self.capture_fault(primary); - if crate::t8122_start::is_t8122(self.device.soc()) { + if crate::m3::t8122_start::is_t8122(self.device.soc()) { let pstate=self.device.pstate_register().ok(); // The job's first GPU start timestamp: nonzero means the firmware dispatched it, // whatever the power state reads at the timeout. let start_stamp=self.jobs[index].batch_gpu_span().ok().map(|span| span[0]); - crate::t8122_start::job_failed_verdict(self.drm.as_ref(),primary,start_stamp,pstate,self.state.health.crashed()); - self.t8122_verdict=true; - } else if crate::t6031_start::is_t6031(self.device.soc()) { - crate::t6031_start::job_failed_verdict(self.drm.as_ref(), primary, self.state.health.crashed()); + crate::m3::t8122_start::job_failed_verdict(self.drm.as_ref().as_ref(),primary,start_stamp,pstate,self.state.health.crashed()); self.t8122_verdict=true; } let events=self.state.event_messages.load(Ordering::Acquire); @@ -116,7 +116,7 @@ impl Inner { dev_err!(self.drm.as_ref(), "M3 engine diagnostics failed: {:?}\n", e); } let _=self.config.log_recovery_state(&self.drm,events); - if crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault) { + if crate::m3::m3_params::g15_debug(crate::m3::m3_params::G15Debug::M3ResumeAfterFault) { if let Err(e)=self.resume_experiment(index, vm) { dev_err!(self.drm.as_ref(),"M3: resume-after-fault: stopped by {:?}\n",e); } @@ -203,10 +203,10 @@ impl Inner { } /// Hardware completion is separate from fence signaling: cancellation can win the /// signal race while the DMA guard must survive until its final published batch retires. - fn packet_active(&self, packet:&Arc)->bool { + fn packet_active(&self, packet:&Arc)->bool { self.active.iter().any(|b|b.entries.iter().any(|(p,_)|Arc::ptr_eq(p,packet))) } - fn retire_packets(&mut self,entries:&[(Arc,usize)]) { + fn retire_packets(&mut self,entries:&[(Arc,usize)]) { for (packet,_) in entries { let pending=self.pending.iter().any(|(p,_)|Arc::ptr_eq(p,packet)); if policy::packet_retired(pending,self.packet_active(packet)) {packet.retired();} @@ -221,37 +221,37 @@ impl Inner { } } /// Signal every packet of `entries` (packets that continue past the batch included). - fn finish_all(entries:&[(Arc,usize)],result:Result) { + fn finish_all(entries:&[(Arc,usize)],result:Result) { for (packet,_) in entries {packet.finish(result);} } /// Choose the next batch from the front of `pending`: the front packet's next commands of /// one engine, and for render also the leading render commands of following packets of the /// same VM, up to the ordered-batch limit. - fn next_batch(&self,max:usize)->Result,usize)>> { + fn next_batch(&self,max:usize)->Result,usize)>> { let mut entries=KVec::new(); let Some((packet,next))=self.pending.first() else {return Ok(entries);}; - let mixed=policy::mixed_prefix(crate::m3_params::mixed_batch(), + let mixed=policy::mixed_prefix(crate::m3::m3_params::mixed_batch(), self.device.soc().chip_id,self.render_initialized,self.active.is_empty(), - packet.commands[*next..].iter().map(|c|matches!(c,crate::m3_submit::Command::Render{..}))); + packet.commands[*next..].iter().map(|c|matches!(c,crate::m3::m3_submit::Command::Render{..}))); if mixed>0 { for index in *next..*next+mixed {entries.push((packet.clone(),index),GFP_KERNEL)?;} return Ok(entries); } - let render=matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}); + let render=matches!(packet.commands[*next],crate::m3::m3_submit::Command::Render{..}); let limit=if render { // The first render draw initializes the shared buffer manager (InitBM) and opens // both queues. Publish it alone: appending later draws first advances the shared // manager and event counters it starts from, and on J613 the second pass's // fragment then fails with firmware error event 4 (draw=2 batch=2). let started=self.jobs.iter().any(|job| matches!(job,NativeJob::Render(j) if j.started())); - if started {crate::m3_params::render_batch_size().min(max).max(1)} else {1} + if started {crate::m3::m3_params::render_batch_size().min(max).max(1)} else {1} } else { - crate::m3_params::compute_batch_size() - .clamp(1,crate::m3_compute_storage::SLOTS) + crate::m3::m3_params::compute_batch_size() + .clamp(1,crate::m3::m3_compute_storage::SLOTS) }; - let same_engine=|c:&crate::m3_submit::Command| render==matches!(c,crate::m3_submit::Command::Render{..}); + let same_engine=|c:&crate::m3::m3_submit::Command| render==matches!(c,crate::m3::m3_submit::Command::Render{..}); let per_pass=self.pass_cost.get(packet.vm.id()); - let budget=crate::m3_params::render_batch_budget_ns(self.device.soc()); + let budget=crate::m3::m3_params::render_batch_budget_ns(self.device.soc()); let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); for (position,(candidate,start)) in self.pending.iter().enumerate() { // Later packets join only render batches, only from their first command, and only @@ -272,19 +272,19 @@ impl Inner { /// Whether the front pending work may be published behind the in-flight batches, and if so /// in which pass slot it starts and how many passes it may use. fn pipeline_slot(&mut self)->Result> { - let Some(last)=self.active.last() else {return Ok(Some((0,crate::m3_pass_layout::SLOTS)));}; + let Some(last)=self.active.last() else {return Ok(Some((0,crate::m3::m3_pass_layout::SLOTS)));}; if !self.render_initialized {return Ok(None);} let Some((packet,next))=self.pending.first() else {return Ok(None);}; - if !matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}) + if !matches!(packet.commands[*next],crate::m3::m3_submit::Command::Render{..}) || self.active.iter().any(|b| b.kind!=0) || !packet.vm.same(&last.entries[0].0.vm) {return Ok(None);} let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); - let Some(room)=policy::pipeline_room(crate::m3_params::pipeline_depth(self.device.soc()), + let Some(room)=policy::pipeline_room(crate::m3::m3_params::pipeline_depth(self.device.soc()), self.active.len(),in_flight,last.base,last.entries.len()) else {return Ok(None);}; // Yield at the next retirement when more urgent work is already waiting. if policy::urgent_waiting(self.pending.iter().map(|(p,_)|p.urgency), last.entries[0].0.urgency) {return Ok(None);} if !policy::within_budget(self.pass_cost.get(packet.vm.id()), - crate::m3_params::render_batch_budget_ns(self.device.soc()),in_flight) {return Ok(None);} + crate::m3::m3_params::render_batch_budget_ns(self.device.soc()),in_flight) {return Ok(None);} if !policy::overlap_ready(self.render_initialized,true,true,true, self.config.pipe_free(0)? && self.config.pipe_free(1)?) {return Ok(None);} Ok(Some(room)) @@ -325,11 +325,8 @@ impl Inner { } Err((error,published))=>{ if published { - if crate::t8122_start::is_t8122(self.device.soc()) && !self.t8122_verdict { - crate::t8122_start::job_setup_failed_verdict(self.drm.as_ref(),error); - self.t8122_verdict=true; - } else if crate::t6031_start::is_t6031(self.device.soc()) && !self.t8122_verdict { - crate::t6031_start::job_setup_failed_verdict(self.drm.as_ref(), error); + if crate::m3::t8122_start::is_t8122(self.device.soc()) && !self.t8122_verdict { + crate::m3::t8122_start::job_setup_failed_verdict(self.drm.as_ref().as_ref(),error); self.t8122_verdict=true; } // A live ring WRITE or doorbell may have been published: keep packets @@ -355,9 +352,9 @@ impl Inner { progress } /// Prepare and publish one batch. On error, reports whether firmware may see its writes. - fn publish(&mut self,entries:&[(Arc,usize)],base:usize,overlap:bool)->core::result::Result { - let render=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Render{..})); - let compute=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Compute(_))); + fn publish(&mut self,entries:&[(Arc,usize)],base:usize,overlap:bool)->core::result::Result { + let render=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3::m3_submit::Command::Render{..})); + let compute=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3::m3_submit::Command::Compute(_))); if render && compute {return self.publish_mixed(entries,base,overlap);} let early=|e:Error|(e,false); let (packet,first_index)=(&entries[0].0,entries[0].1); @@ -378,19 +375,19 @@ impl Inner { if !self.packets.iter().any(|q| Arc::ptr_eq(q,p)) {self.packets.push(p.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} } let found=self.jobs.iter().position(|job| matches!((job,control), - (NativeJob::Compute(_),crate::m3_submit::Command::Compute(_)) | - (NativeJob::Render(_),crate::m3_submit::Command::Render{..}))); + (NativeJob::Compute(_),crate::m3::m3_submit::Command::Compute(_)) | + (NativeJob::Render(_),crate::m3::m3_submit::Command::Render{..}))); let index=found.unwrap_or(self.jobs.len()); if found.is_none() { let job=match control { - crate::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), - crate::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), }; self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; } else { match (&mut self.jobs[index],control) { - (NativeJob::Render(_),crate::m3_submit::Command::Render{..})=>{}, - (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>j.replay(&self.uat,&packet.vm,c).map_err(early)?, + (NativeJob::Render(_),crate::m3::m3_submit::Command::Render{..})=>{}, + (NativeJob::Compute(j),crate::m3::m3_submit::Command::Compute(c))=>j.replay(&self.uat,&packet.vm,c).map_err(early)?, _=>return Err(early(ENOTSUPP)), } } @@ -398,11 +395,11 @@ impl Inner { let prepared=(||->Result { for (slot,(p,next)) in entries.iter().enumerate() { if slot!=0 { - let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; j.append(c)?; } let timestamps=p.timestamp_addresses(*next); - let coalesce=crate::m3_compute_storage::coalesce_stamp_flush( + let coalesce=crate::m3::m3_compute_storage::coalesce_stamp_flush( p.wide_visibility[*next],timestamps,slot+1==batch_count); j.prepare_completion(coalesce)?; j.set_user_timestamps(timestamps)?; @@ -417,8 +414,8 @@ impl Inner { } } let mut coalesced_stamps=0; - let coalesce_render=crate::m3_params::render_stamp_coalesce(); - let internal_timestamps=crate::m3_params::render_internal_timestamps(); + let coalesce_render=crate::m3::m3_params::render_stamp_coalesce(); + let internal_timestamps=crate::m3::m3_params::render_internal_timestamps(); let chip=self.device.soc().chip_id; if let NativeJob::Render(j)=&mut self.jobs[index] { let mut commands=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?; @@ -426,7 +423,7 @@ impl Inner { j.begin_batch(&self.drm,&self.uat,&packet.vm,&commands,base,overlap).map_err(early)?; let prepared=(||->Result { for (slot,(p,next)) in entries.iter().enumerate() { - let crate::m3_submit::Command::Render{command,usc}=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command,usc}=p.commands[*next] else {return Err(EINVAL);}; let timestamps=p.render_timestamp_addresses(*next); let coalesce=policy::coalesce_render_stamps(coalesce_render,chip, overlap,slot,batch_count,timestamps); @@ -448,6 +445,19 @@ impl Inner { } } let preparation_ns=prepare.elapsed().as_nanos(); + // The batch's kernel objects may have created leaf tables in the host window: link + // them into the firmware's shared L2 before it is told about the batch. A failure + // means the window no longer matches what the firmware may walk: stop scheduling and + // retain the batch, as for a published error. + if let Err(error)=self.window.sync_all(&self.uat) { + return Err((error,true)); + } + // The command aliases live in the context-0 lower tree, which has no bind slot on the + // adopted UAT (7.1.12 gave it the fixed slot 0), so remapping one invalidates nothing: + // drop context 0's non-global entries before the firmware fetches the new ones. + crate::mem::sync(); + crate::mem::tlbi_asid(0); + crate::mem::sync(); self.t8122_verdict=false; let previous_events=self.config.completed_events; let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; @@ -470,7 +480,7 @@ impl Inner { }; let measure=crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary); let (base,last_draw)=match &self.jobs[index] {NativeJob::Render(j)=>{let (b,_,d)=j.batch();(b,d)},_=>(0,0)}; - Ok(Batch { entries:owned, index, kind:usize::from(matches!(control,crate::m3_submit::Command::Compute(_))), + Ok(Batch { entries:owned, index, kind:usize::from(matches!(control,crate::m3::m3_submit::Command::Compute(_))), first:control, preparation_ns, previous_events, expected_events, start:Instant::::now(), measure, doorbell_counter_ns:if measure {physical_counter() as i64 * 1000 / 24} else {0}, polling_ns:0, polls:0, stamp_ns:-1, first_event_ns:-1, events_ns:-1, completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0, coalesced_stamps, base, last_draw, publication_tick, mixed:None }) @@ -478,7 +488,7 @@ impl Inner { /// Prepare both idle queues from one packet. Cross-engine changes wait on /// the preceding command's firmware stamp; all commands remain GPU ordered. /// Producer storage and DMA guards survive every partial publication error. - fn publish_mixed(&mut self,entries:&[(Arc,usize)], + fn publish_mixed(&mut self,entries:&[(Arc,usize)], base:usize,overlap:bool)->core::result::Result { let early=|e:Error|(e,false); if overlap || base!=0 || self.device.soc().chip_id!=0x6030 @@ -486,7 +496,7 @@ impl Inner { let packet=&entries[0].0; if entries.iter().any(|(p,_)|!Arc::ptr_eq(packet,p)) {return Err(early(EINVAL));} let prepare=Instant::::now(); - let internal_timestamps=crate::m3_params::render_internal_timestamps(); + let internal_timestamps=crate::m3::m3_params::render_internal_timestamps(); self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; self.jobs.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; self.packets.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; @@ -497,8 +507,8 @@ impl Inner { for (p,i) in entries { owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?; match p.commands[*i] { - c@crate::m3_submit::Command::Render{..}=>render_commands.push(c,GFP_KERNEL).map_err(|_|early(ENOMEM))?, - crate::m3_submit::Command::Compute(c)=>{computes+=1;first_compute.get_or_insert(c);}, + c@crate::m3::m3_submit::Command::Render{..}=>render_commands.push(c,GFP_KERNEL).map_err(|_|early(ENOMEM))?, + crate::m3::m3_submit::Command::Compute(c)=>{computes+=1;first_compute.get_or_insert(c);}, } } let renders=render_commands.len(); @@ -514,7 +524,7 @@ impl Inner { let prepared=(||->Result { let control=first_compute.ok_or(EINVAL)?; if existing.is_none() { - let job=crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, + let job=crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, self.config.stats_region()?,control,self.device.soc().registers)?; self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?; } else { @@ -528,12 +538,12 @@ impl Inner { let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; j.begin_batch(&self.drm,&self.uat,&packet.vm,&render_commands,0,false)?; render_prepared=true; - let mut previous:Option<(bool,crate::m3_sync_layout::CompletionDependency)>=None; + let mut previous:Option<(bool,crate::m3::m3_sync_layout::CompletionDependency)>=None; let mut compute_slot=0; let mut dependencies=0; for (p,i) in entries { match p.commands[*i] { - crate::m3_submit::Command::Render{command,usc}=>{ + crate::m3::m3_submit::Command::Render{command,usc}=>{ let predecessor=previous.filter(|(render,_)|!*render).map(|(_,d)|d); let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; let timestamps=p.render_timestamp_addresses(*i); @@ -543,7 +553,7 @@ impl Inner { dependencies+=usize::from(predecessor.is_some()); previous=Some((true,j.completion_dependency()?)); }, - crate::m3_submit::Command::Compute(c)=>{ + crate::m3::m3_submit::Command::Compute(c)=>{ let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; if compute_slot!=0 {j.append(c)?;} if let Some((true,dependency))=previous { @@ -628,14 +638,14 @@ impl Inner { let mut compute_slot=0; for (packet,next) in batch.entries.iter() { match packet.commands[*next] { - crate::m3_submit::Command::Render{command:r,..}=>{ + crate::m3::m3_submit::Command::Render{command:r,..}=>{ let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; let s=j.slot_stage_ns(0,mixed.renders,render_slot)?; dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} mixed=1 slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n", vm,render_slot,mixed.renders,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); render_slot+=1; }, - crate::m3_submit::Command::Compute(c)=>{ + crate::m3::m3_submit::Command::Compute(c)=>{ let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; let ns=j.slot_gpu_ns(compute_slot)?; dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} mixed=1 slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n", @@ -718,10 +728,10 @@ impl Inner { } } } - let power_enabled=crate::m3_params::active_power(); + let power_enabled=crate::m3::m3_params::active_power(); if power_enabled { let since_last=self.active_power_last.map(|last|last.elapsed().as_nanos()); - if crate::m3_active_power::due(power_enabled,self.device.soc().chip_id, + if crate::m3::m3_active_power::due(power_enabled,self.device.soc().chip_id, stamped,self.active_power_samples,since_last) { self.active_power_last=Some(Instant::::now()); self.active_power_samples+=1; @@ -760,23 +770,20 @@ impl Inner { (evented,batch.start.elapsed()>=Delta::from_secs(2),batch.previous_events) }; if stamped && evented { - let retire_mmio=crate::m3_params::retire_mmio(self.device.soc()); + let retire_mmio=crate::m3::m3_params::retire_mmio(self.device.soc()); match self.device.check_idle_parts( - retire_mmio&crate::m3_params::RETIRE_MMIO_BUSY!=0, - retire_mmio&crate::m3_params::RETIRE_MMIO_FAULTS!=0) { + retire_mmio&crate::m3::m3_params::RETIRE_MMIO_BUSY!=0, + retire_mmio&crate::m3::m3_params::RETIRE_MMIO_FAULTS!=0) { Ok(())=>match self.config.pipes_idle() { Ok(true)=>{ if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { let tick=physical_counter(); if let Some(batch)=self.active.first_mut() {batch.pipes_idle_tick=tick;} } - if retire_mmio&crate::m3_params::RETIRE_MMIO_PSTATE!=0 { + if retire_mmio&crate::m3::m3_params::RETIRE_MMIO_PSTATE!=0 { if let Err(e)=self.config.check_pstate(&self.drm,&self.device,"after a job") { - if crate::t8122_start::is_t8122(self.device.soc()) { - crate::t8122_start::cap_violated_verdict(self.drm.as_ref(),e); - self.t8122_verdict=true; - } else if crate::t6031_start::is_t6031(self.device.soc()) { - crate::t6031_start::cap_violated_verdict(self.drm.as_ref(), e); + if crate::m3::t8122_start::is_t8122(self.device.soc()) { + crate::m3::t8122_start::cap_violated_verdict(self.drm.as_ref().as_ref(),e); self.t8122_verdict=true; } self.fail_active(e);return Err(e); @@ -862,8 +869,8 @@ impl Inner { if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { let observed_tick=physical_counter(); let (cs,span)=match (&mut self.jobs[index],control) { - (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>(c.base,j.batch_gpu_span()?), - (NativeJob::Render(j),crate::m3_submit::Command::Render{command:r,..})=>{let (a,b)=j.gpu_ticks(batch.base,batch_count)?;(r.vdm_base,[a,b])}, + (NativeJob::Compute(j),crate::m3::m3_submit::Command::Compute(c))=>(c.base,j.batch_gpu_span()?), + (NativeJob::Render(j),crate::m3::m3_submit::Command::Render{command:r,..})=>{let (a,b)=j.gpu_ticks(batch.base,batch_count)?;(r.vdm_base,[a,b])}, _=>return Err(EINVAL), }; let valid=batch.publication_tick!=0 && span[0]>=batch.publication_tick && span[1]>=span[0] && observed_tick>=span[1]; @@ -874,14 +881,14 @@ impl Inner { if kind==0 {gpu_ns} else {0},stages[1]); if let NativeJob::Compute(j)=&mut self.jobs[index] { for (slot,(p,next)) in batch.entries.iter().enumerate() { - let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; let ns=j.slot_gpu_ns(slot)?; dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n",p.vm.id(),slot,batch_count,c.base,c.end,c.usc_base,ns); } } if let NativeJob::Render(j)=&mut self.jobs[index] { for (slot,(p,next)) in batch.entries.iter().enumerate() { - let crate::m3_submit::Command::Render{command:r,..}=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=p.commands[*next] else {return Err(EINVAL);}; let s=j.slot_stage_ns(batch.base,batch_count,slot)?; dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n",p.vm.id(),slot,batch_count,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); } @@ -893,7 +900,7 @@ impl Inner { preparation_ns, active_ns, gpu_ns, stages); let retirement_ns=retire_start.map_or(0,|t|t.elapsed().as_nanos()); if batch.measure { - if let crate::m3_submit::Command::Render {command:r,..}=control { + if let crate::m3::m3_submit::Command::Render {command:r,..}=control { if batch_count>1 {self.geometry_overflow+=1;} else { if let Some(g)=self.geometry.iter_mut().find(|g| g.count==0 || (g.width==r.width && g.height==r.height && g.samples==r.samples)) { @@ -917,9 +924,9 @@ impl Inner { } if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { match control { - crate::m3_submit::Command::Render {command:r,..}=> + crate::m3::m3_submit::Command::Render {command:r,..}=> dev_info!(self.drm.as_ref(),"M3_PASS kind=render batch={} vdm={:#x} width={} height={} samples={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={}\n",batch_count,r.vdm_base,r.width,r.height,r.samples,preparation_ns,active_ns,gpu_ns,stages[0],stages[1],stages[2]), - crate::m3_submit::Command::Compute(_)=> + crate::m3::m3_submit::Command::Compute(_)=> dev_info!(self.drm.as_ref(),"M3_PASS kind=compute prepare_ns={} active_ns={} gpu_ns={}\n",preparation_ns,active_ns,gpu_ns), } } @@ -927,14 +934,12 @@ impl Inner { t[0]+=1;t[1]+=preparation_ns;t[2]+=active_ns;t[3]+=gpu_ns as i64; for i in 0..3 {t[4+i]+=stages[i] as i64;} // Running totals are in debugfs `timing`; the periodic line is a SubmitTiming diagnostic. - if t[0]==1 && crate::t8122_start::is_t8122(self.device.soc()) { + if t[0]==1 && crate::m3::t8122_start::is_t8122(self.device.soc()) { let span=match &mut self.jobs[index] { NativeJob::Compute(j)=>j.batch_gpu_span().ok(), NativeJob::Render(j)=>j.gpu_ticks(batch.base,batch_count).ok().map(|(a,b)|[a,b]), }; - crate::t8122_start::job_completed_verdict(self.drm.as_ref(),kind,gpu_ns,span); - } else if t[0]==1 && crate::t6031_start::is_t6031(self.device.soc()) { - crate::t6031_start::job_completed_verdict(self.drm.as_ref(), kind, gpu_ns); + crate::m3::t8122_start::job_completed_verdict(self.drm.as_ref().as_ref(),kind,gpu_ns,span); } if t[0]%128==0 && crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { dev_info!(self.drm.as_ref(),"M3_TIMING kind={} count={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} ordinal={}\n",kind,t[0],t[1],t[2],t[3],t[4],t[5],t[6], @@ -971,26 +976,23 @@ impl Inner { pub(crate) struct Runtime { inner: ManuallyDrop> } impl Runtime { #[inline(never)] - pub(crate) fn new(pdev: &platform::Device, device: Device, contents: crate::m3_adt_config::Contents) -> Result { + pub(crate) fn new(pdev: &platform::Device>, drm: &driver::AsahiDrmDevice, device: Device, contents: crate::m3::m3_adt_config::Contents) -> Result { // On an armed T8122, name a setup failure before the GPU coprocessor is started (RTKit // state, mailbox transport, the DRM device). Captured as a Copy bool so the closure does // not borrow `device`, which is moved into Inner below. - let t8122 = crate::t8122_start::is_t8122(device.soc()); - let t6031 = crate::t6031_start::is_t6031(device.soc()); + let t8122 = crate::m3::t8122_start::is_t8122(device.soc()); let coproc_refused = |e: &Error| { - if t8122 { crate::t8122_start::coproc_setup_refused(pdev.as_ref(), *e); } - if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), *e); } + if t8122 { crate::m3::t8122_start::coproc_setup_refused(pdev.as_ref(), *e); } }; device.require_stopped(pdev).inspect_err(coproc_refused)?; // Reserve before starting ASC. ENOMEM must not drop firmware owners // while the coprocessor may still access them. Writing the completed // graph into this allocation is infallible. let owner = KBox::::new_uninit(GFP_KERNEL)?; - let drm: driver::AsahiDevRef = kernel::drm::Device::new(pdev.as_ref(), driver::AsahiData::new(pdev, None, true)).inspect_err(coproc_refused)?; + let drm: driver::AsahiDevRef = drm.into(); let state = m3_rtkit::State::new(pdev, drm.clone(), device.firmware().resources.regions[5]).inspect_err(coproc_refused)?; let mut transport = rtkit::RtKit::new(pdev.as_ref(), None, 0, state.clone()).inspect_err(coproc_refused)?; - if crate::m3_adt_config::stop_before_asc(pdev.as_ref()) { - if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), ENODEV); } + if crate::m3::m3_adt_config::stop_before_asc(pdev.as_ref()) { return Err(ENODEV); } // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. @@ -1004,18 +1006,22 @@ impl Runtime { Pin::new(&mut transport).start_endpoint(ep)?; } started=true; - // The UAT geometry of the admitted SoC (complete: `Soc::require_complete`). - // SAFETY: the admitted RTKit (J514S on T6030) is awake; no initdata or GPU - // job has been published. Its running PPL handoff owns the peer lock. - let uat=unsafe {mmu::Uat::new_m3_running(&drm,device.soc())}?; + // The firmware's UAT, adopted as the G15 backend adopts it on 7.2: the handoff and + // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as + // every context's upper half, and the host window (root entry 2) through the + // shared L2. The RTKit is awake and no initdata or GPU job has been published. + let (uat,mut window)=adopt_uat(&drm,device.soc(),&device.firmware().resources)?; let config=Config::new(&drm,&uat,device.firmware(),&contents)?; - Ok((uat,config)) + // Link every kernel table the configuration created before the firmware sees an + // address in it, then drop any translation cached before they existed. + window.sync_all(&uat)?; + crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + Ok((uat,window,config)) })(); - let (uat,config)=match prepared { + let (uat,window,config)=match prepared { Ok(v)=>v, Err(e)=>{ - crate::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); - if t6031 { crate::t6031_start::prepare_verdict(pdev.as_ref(), started, e); } + crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; @@ -1025,34 +1031,43 @@ impl Runtime { return Err(e); } }; - Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, drm, device, - jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3_pass_layout::SLOTS], + Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, window, drm, device, + jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3::m3_pass_layout::SLOTS], geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false, - boot_step:crate::t8122_start::BootStep::Publish })) }) + boot_step:crate::m3::t8122_start::BootStep::Publish })) }) } pub(crate) fn drm(&self) -> driver::AsahiDevRef { self.inner.drm.clone() } pub(crate) fn health(&self) -> Arc { self.inner.state.health.clone() } + /// The RTKit callbacks' state, which owns the completion worker. + pub(crate) fn rtkit_state(&self) -> Arc { self.inner.state.clone() } pub(crate) fn new_vm(&mut self, id: u64, range: core::ops::Range) -> Result { use crate::util::RangeExt; let reserved_range=0x70_0000_0000..0x80_0000_0000; if range.overlaps(reserved_range.clone()) {return Err(EINVAL);} if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } - let vm=self.inner.uat.new_vm(id, range)?; + // Status and job lifetime tracking: deferred unmaps while accepted work runs + // (7.1.12's M3 VM job lifetime, the shared `mmu::lifetime` on 7.2). + let vm=self.inner.uat.new_vm(id, range)?.with_tracked_status()?; let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?; vm.install_driver_mappings(KVec::new(),reserved)?; Ok(vm) } - pub(crate) fn map_timestamp(&self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { + pub(crate) fn map_timestamp(&mut self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } - bo.map_range_into_range(self.inner.uat.kernel_vm(),range,crate::agx_memory::TIMESTAMP_RANGE, - mmu::UAT_PGSZ as u64,mmu::PROT_FW_SHARED_RW,false) + let mapping=bo.map_range_into_range(self.inner.uat.kernel_vm(),range,crate::m3::agx_memory::TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64,mmu::PROT_FW_SHARED_RW,false)?; + // A job of an overlapping batch can reach the firmware without a new publication: + // link a new leaf table now, not only at the next batch. + let inner: &mut Inner=&mut self.inner; + inner.window.sync_all(&inner.uat)?; + Ok(mapping) } /// Queue a packet and return its fence at once. The scheduler can hand over further /// ready jobs up to its credit limit while the GPU runs. Only the completion worker /// publishes: it coalesces render commands of one VM and urgency, and may keep bounded /// same-VM batches in flight after InitBM retirement. VM/engine changes wait for full /// retirement. A burst can therefore share a firmware round trip across packets. - pub(crate) fn submit(shared:&crate::m3_drm::Shared,packet:Arc)->Result { + pub(crate) fn submit(shared:&crate::m3::m3_drm::Shared,packet:Arc)->Result { crate::debug::update_debug_flags(); let mut guard=shared.lock(); let inner=&mut *Option::as_mut(&mut *guard).ok_or(ENODEV)?.inner; @@ -1069,14 +1084,14 @@ impl Runtime { if inner.pending.len()>=policy::MAX_PACKETS {return Err(EBUSY);} inner.pending.reserve(1,GFP_KERNEL)?; inner.pending.insert_within_capacity(at,(packet,0)).map_err(|_|ENOMEM)?; - let drm=inner.drm.clone(); + let state=inner.state.clone(); drop(guard); - crate::m3_completion::queue(&drm); + crate::m3::m3_completion::queue(&state); Ok(()) } /// Withdraw a packet's unpublished remainder on cancellation or timeout. Its active /// commands remain owned until retirement; error signaling cannot release a DMA guard. - pub(crate) fn withdraw(shared:&crate::m3_drm::Shared,packet:&Arc)->bool { + pub(crate) fn withdraw(shared:&crate::m3::m3_drm::Shared,packet:&Arc)->bool { let mut guard=shared.lock(); let Some(runtime)=Option::as_mut(&mut *guard) else {return false;}; let inner: &mut Inner=&mut runtime.inner; @@ -1120,18 +1135,17 @@ impl Runtime { Err(_)=>{inner.fail_pending(EIO);Service::Idle}, } } - pub(crate) fn boot(&mut self, pdev: &platform::Device) -> Result { - let is_t8122 = crate::t8122_start::is_t8122(self.inner.device.soc()); - let is_t6031 = crate::t6031_start::is_t6031(self.inner.device.soc()); - self.inner.boot_step = crate::t8122_start::BootStep::Publish; + pub(crate) fn boot(&mut self, pdev: &platform::Device>) -> Result { + let is_t8122 = crate::m3::t8122_start::is_t8122(self.inner.device.soc()); + self.inner.boot_step = crate::m3::t8122_start::BootStep::Publish; let result = self.boot_inner(pdev); if let Err(error) = result { // On T8122, log the firmware readiness words on every boot failure, so the verdict's // "see M3 firmware readiness" always has a line to point at. The ready-wait timeout in // boot_inner already logs them (on every SoC, as in 12.0), so skip that case here. - let logged = self.inner.boot_step == crate::t8122_start::BootStep::AwaitReady + let logged = self.inner.boot_step == crate::m3::t8122_start::BootStep::AwaitReady && error == ETIMEDOUT; - if (is_t8122 || is_t6031) && !logged { + if is_t8122 && !logged { let inner: &mut Inner = &mut *self.inner; let _ = inner.config.log_ready(&inner.drm); } @@ -1139,18 +1153,13 @@ impl Runtime { } if is_t8122 { let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); - crate::t8122_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, - self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); - } - if is_t6031 { - let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); - crate::t6031_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, + crate::m3::t8122_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); } result } - fn boot_inner(&mut self, pdev: &platform::Device) -> Result { - use crate::t8122_start::BootStep; + fn boot_inner(&mut self, pdev: &platform::Device>) -> Result { + use crate::m3::t8122_start::BootStep; let root = self.inner.config.root(); dev_info!(pdev.as_ref(), "M3: publishing owned initdata {:#x}\n", root); self.inner.boot_step = BootStep::Publish; @@ -1174,7 +1183,7 @@ impl Runtime { inner.config.drain(&inner.drm)?; inner.config.log_ready(&inner.drm)?; if inner.device.soc().chip_id==0x6030 && - (crate::m3_params::active_power() || crate::m3_params::globals_debug_off()) { + (crate::m3::m3_params::active_power() || crate::m3::m3_params::globals_debug_off()) { inner.config.log_globals_debug(&inner.drm)?; } inner.config.check_pstate(&inner.drm, &inner.device, "after boot")?; @@ -1244,3 +1253,22 @@ pub(crate) fn physical_counter() -> u64 { if frequency == 0 { return 0; } (tick / frequency) * 24_000_000 + (tick % frequency) * 24_000_000 / frequency } + +/// Adopts the running firmware's UAT on `soc`: the `mmu::Uat` of the admitted reserved regions +/// (ttbs, pagetables, handoff, shared-l2) and the PPL exchange with its startup epoch, as the +/// G15 backend does it on 7.2 (`g15::uat`). +fn adopt_uat(drm: &driver::AsahiDrmDevice, soc: &'static crate::m3::m3_soc::Soc, + resources: &crate::m3::agx_resources::Resources) -> Result<(mmu::Uat, crate::g15::uat::Window)> { + let cfg = soc.hwcfg.ok_or(ENODEV)?; + let [ttbs, tables, handoff, shared_l2, ..] = resources.regions; + let range = |r: crate::m3::agx_resources::Region| r.base..r.base + r.size; + let geometry = mmu::UatGeometry { ias: cfg.uat_ias.into(), oas: cfg.uat_oas }; + let regions = mmu::AdoptedRegions { handoff: range(handoff), ttbs: range(ttbs), kernel_root: tables.base }; + let uat = mmu::Uat::new_adopted(drm, geometry, ®ions)?; + let mut window = crate::g15::uat::Window::from_regions(handoff.base, ttbs.base, tables.base, shared_l2.base)?; + window.bootstrap(drm.as_ref().as_ref(), &uat)?; + // 7.1.12 cleaned every reserved-table write to the point of coherency; these regions may be + // reserved System RAM mapped cacheable. + window.clean_tables(); + Ok((uat, window)) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_soc.rs b/drivers/gpu/drm/asahi/m3/m3_soc.rs index f19717d536f72..7be649415cccc 100644 --- a/drivers/gpu/drm/asahi/m3/m3_soc.rs +++ b/drivers/gpu/drm/asahi/m3/m3_soc.rs @@ -2,48 +2,230 @@ //! The per-SoC facts of the M3 runtime backend. //! -//! Each table describes the configuration of one GPU. This compile-only subset holds only the -//! facts the M3 boundary carries; the register windows, memory layout, firmware images and -//! admission of the full table stay with the M3 runtime port. Nothing here is read by a probe. - -use super::m3_compute_layout::RegisterSet; - -/// The HwDataB words of the runtime backend's InitData that are neither its GPU VA layout nor -/// device-tree values: configuration words where the manager layout keeps 0 or its validated -/// values (including the A/B pair +0xa98/+0xb40 at 0), and the two unit masks. They are values -/// of one SoC's runtime InitData (`m3_soc::Soc::hwdata_b`), not derived from the device tree. -#[derive(Debug)] -pub(crate) struct G15RuntimeHwDataB { - pub(crate) unk_454: u32, - pub(crate) unk_464: u32, - pub(crate) unk_a7c: u64, - pub(crate) unk_a98: u32, - pub(crate) unk_abc: u64, - pub(crate) unk_ae4: u32, - pub(crate) unk_b20: u32, - pub(crate) unk_b24: u64, - /// +0xb40, the "UAT enabled" word of the manager layout. - pub(crate) unk_554: u32, - pub(crate) unk_17b8: u32, - /// Unit enable masks, +0x17c0/+0x17c8. - pub(crate) unit_mask_a: u64, - pub(crate) unit_mask_b: u32, - pub(crate) unk_1808: u32, - pub(crate) unk_1818: u32, +//! Each table describes the register windows, memory layout and configuration of one GPU. +//! Missing configuration is `None`; outstanding runtime requirements are listed in `unported`. +//! Admission rejects either before reading firmware, mapping GPU registers or starting the +//! GPU coprocessor ([`Soc::require_complete`]). + +use kernel::{c_str, device, prelude::*}; + +use crate::{ + m3::hw, + m3::initdata::G15RuntimeHwDataB, + m3::m3_compute_layout::RegisterSet, + m3::m3_board::KnownImage, + m3::m3_firmware::Layout, + m3::m3_init_storage::IoMap, // +}; + +/// The SGX identification words the runtime admits, each compared under its mask. +pub(crate) struct IdWords { + /// SGX+0xd04000: family [31:24], variant [23:16], revision [15:8]. + pub(crate) version: u32, + pub(crate) version_mask: u32, + /// SGX+0xd04010: dies [19:16], clusters per die [15:8]. + pub(crate) counts: u32, + pub(crate) counts_mask: u32, } +/// The register windows the runtime admits, as CPU physical addresses (device-tree `reg` values +/// translated through `arm-io` `ranges`). +pub(crate) struct Windows { + /// Base of the GPU coprocessor window (`asc`, ADT `gfx-asc` reg[0]). + pub(crate) asc: u64, + /// Base of the GPU window (`sgx`, ADT `sgx` reg[0]). + pub(crate) sgx: u64, + /// Base of the GPU coprocessor mailbox, a 16 KiB window inside `asc`. + pub(crate) mailbox: u64, +} + +/// The performance-state table the runtime accepts: checked on the generated InitData before +/// any GPU register is touched, and again on the uploaded HwDataB. +#[derive(Copy, Clone, Debug)] +pub(crate) enum PstateTable { + /// A fixed table: `states` states above the off state, the highest at `top_mhz` MHz. + Fixed { states: u32, top_mhz: u32 }, + /// The table of the device tree's operating points: one state per distinct voltage above + /// the off state, the highest at the highest frequency + /// (`t8122_admission::opp_table_shape`). + DeviceTree, +} + +/// Optional userspace features advertised for a SoC (`DRM_ASAHI_GET_PARAMS`). The runtime +/// implements them the same way on every SoC; a SoC advertises one once it has been validated +/// there. Soft faults are never advertised: the runtime does not apply `asahi.fault_control`. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Features { + /// `DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY`. + pub(crate) compute_wide_visibility: bool, + /// `DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY`, while `asahi.m3_early_tiling` is set. + pub(crate) fragment_dependency: bool, +} + +/// The MTR temperature-sensor masks of the runtime's HwDataA, where the SoC table sets them +/// instead of the hardware configuration. +#[derive(Copy, Clone, Debug)] +pub(crate) struct MtrMasks { + /// The fast-die sensor mask, in both HwDataA copies (+0x8ac and +0x1288). + pub(crate) fast_die: u64, + /// The mask the firmware matches an MTR alarm against (HwDataA +0x1a98). An alarm from a + /// sensor outside it, or any alarm while it is 0, is fatal to the firmware. + pub(crate) alarm: u64, +} + +/// Where the boot loader exports a SoC's decoded GPU leakage fuse values, in `/chosen`, and the +/// boot loader switch that makes them this boot's leakage. +#[derive(Copy, Clone, Debug)] +pub(crate) struct LeakFuse { + /// Two cells, value 1 then value 2, encoded as `apple,core-leak-coef` is. + pub(crate) values: &'static CStr, + /// The switch: the boot loader uses the values exactly when it holds the string "1" (the + /// two bytes `1\0`), the form it copies from its configuration line. + pub(crate) switch: &'static CStr, +} + +/// Where the runtime places the HwData object (`m3_init_storage::HARDWARE_DATA`), when it is not +/// the fixed allocation's address, and how the firmware maps it. +#[derive(Copy, Clone, Debug)] +pub(crate) struct HwDataObject { + /// Firmware VA of the object. + pub(crate) address: u64, + /// Map the object cacheable for the firmware (its atomic updates need cacheable memory) + /// instead of uncached. + pub(crate) cached: bool, +} + +/// One firmware IO mapping of the runtime's InitData: HwDataB slot, physical address, total +/// size, element size, writable. +pub(crate) type IoMapping = (usize, u64, u32, u32, bool); + +/// The M3 runtime's facts for one SoC. pub(crate) struct Soc { /// SoC name, for logs. pub(crate) name: &'static str, + /// GPU name, for logs. + pub(crate) gpu_name: &'static str, + pub(crate) chip_id: u32, + pub(crate) gpu_variant: hw::GpuVariant, + pub(crate) gpu_revision: hw::GpuRevision, + pub(crate) gpu_revision_id: hw::GpuRevisionID, + /// Root node compatible of the SoC. + pub(crate) board: &'static str, + /// GPU node compatible. + pub(crate) gpu: &'static str, + /// Boards the runtime starts on by default and registers a render node on. On a T8122 it + /// starts on no other board, whatever `asahi.m3_backend` says (`m3_params::t8122_backend`). + pub(crate) validated_boards: &'static [&'static [u8]], + /// GPU clusters, and core slots per cluster. + pub(crate) clusters: u32, + pub(crate) cores_per_cluster: u32, + /// Accepted compatible lists of the GPU coprocessor mailbox. + pub(crate) mailbox_compatibles: &'static [&'static [u8]], + /// The mailbox interrupts: send-empty, send-not-empty, recv-empty, recv-not-empty, as the + /// mailbox node's `interrupts` cells (three per interrupt, or four with the die cell of a + /// multi-die interrupt controller). + pub(crate) mailbox_interrupts: &'static [u32], + /// The register windows of the GPU node and its mailbox. + pub(crate) windows: Windows, + /// The identification words the runtime admits. + pub(crate) id: IdWords, + /// GPU firmware images the runtime can identify. + pub(crate) images: &'static [KnownImage], + /// The layout of the loaded firmware the runtime accepts. + pub(crate) firmware: Option<&'static Layout>, + /// The hardware configuration of the InitData. + pub(crate) hwcfg: Option<&'static hw::HwConfig>, + /// The firmware IO mappings of the InitData. + pub(crate) io_mappings: Option<&'static [IoMapping]>, + /// The runtime's mapping of each of those IO mappings: the CPU physical block and the + /// firmware VA it is mapped at. One per entry of `io_mappings`, covering it. + pub(crate) iomaps: Option<&'static [IoMap]>, + /// The performance-state table the runtime accepts. + pub(crate) pstates: PstateTable, + /// The SGX write (offset, value) made after the identity checks, before the firmware starts. + pub(crate) sgx_setup: Option<(usize, u32)>, /// The runtime InitData's HwDataB configuration words and unit masks. pub(crate) hwdata_b: Option<&'static G15RuntimeHwDataB>, + /// The optional userspace features advertised. + pub(crate) features: Features, + /// Whether the power-management coefficients, the leakage coefficients and the operating + /// points are this machine's, added by the boot loader from its ADT, rather than static + /// device-tree values. Admission then requires every one of them (see + /// `m3_board::check_boot_loader_power`), with no driver default standing in except the + /// idle-off standby timer, which an ADT may lack. + pub(crate) power_from_boot_loader: bool, + /// Code the runtime runs after admission that still holds another SoC's values. + pub(crate) unported: &'static [&'static str], + /// The GPU registers read after every job by default (`asahi.m3_retire_mmio` bits). + pub(crate) retire_mmio: u64, + /// The MTR sensor masks, when they are not the hardware configuration's. + pub(crate) mtr_masks: Option, + /// The HwData object's placement and mapping, when they are not the fixed allocation's. + pub(crate) hwdata_object: Option, /// The register lists the GPU's firmware expects in compute and render commands. pub(crate) registers: RegisterSet, + /// Globals +0x7d0, which gates the firmware's frequency-feedback cap: the firmware measures + /// the GPU clock against the requested performance state and, while this is 1, caps the + /// state on a shortfall (`asahi.m3_ut_engagement` overrides it). + pub(crate) ut_engagement: u32, + /// The highest GPU power target, in mW, while the boot loader's power model is a stand-in: + /// the operating points' powers are scaled down together so that the highest is this. + pub(crate) power_target_cap_mw: Option, /// Words of the generated HwData object (offset into the object, value) that this SoC's /// firmware takes with other values than the shared builder writes. pub(crate) hwdata_words: &'static [(usize, u32)], /// The same for the Globals object. pub(crate) globals_words: &'static [(usize, u32)], + /// The boot loader's decoded GPU leakage fuse, when it exports one for this SoC. + pub(crate) leak_fuse: Option, + /// Whether each operating point may give each cluster its own voltage (the per-cluster, + /// binned voltage tables of T6031). Otherwise every cluster of a point must have the same. + pub(crate) per_cluster_voltages: bool, +} + +impl Soc { + /// Reject incomplete configuration before accessing the GPU. + pub(crate) fn require_complete(&self, dev: &device::Device) -> Result { + let mut missing = 0u32; + let mut note = |what: &str| { + dev_info!(dev, "M3 {}: missing {}\n", self.gpu_name, what); + missing += 1; + }; + if !self.images.iter().any(|image| image.initdata_magic.is_some()) { + note("the identity of the loaded GPU firmware image and its InitData version"); + } + if self.firmware.is_none() { + note("the layout of the loaded GPU firmware (segment sizes and VAs)"); + } + if self.hwcfg.is_none() { + note("the hardware configuration of the InitData (HwConfig)"); + } + if self.io_mappings.is_none() { + note("the firmware IO mappings of the InitData"); + } + if self.iomaps.is_none() { + note("the runtime's mapping of the firmware IO mappings (IO maps)"); + } + if self.sgx_setup.is_none() { + note("the SGX setup write made before the firmware starts"); + } + if self.hwdata_b.is_none() { + note("the runtime HwDataB configuration words and unit masks"); + } + for &what in self.unported { + note(what); + } + if missing == 0 { + return Ok(()); + } + dev_err!( + dev, + "M3 {}: {} configuration requirements missing; GPU startup disabled\n", + self.gpu_name, + missing + ); + Err(ENODEV) + } } /// The HwDataB configuration words and unit masks of the T6030 runtime InitData, as checked on @@ -77,24 +259,115 @@ pub(crate) static T8122_HWDATA_B: G15RuntimeHwDataB = G15RuntimeHwDataB { }; /// Every SoC the M3 runtime has a table for. -pub(crate) static SOCS: [&Soc; 2] = [&T6030, &T8122]; - -/// T6030 (M3 Pro, G15S): one die, two clusters of ten core slots. -pub(crate) static T6030: Soc = Soc { - name: "T6030", - hwdata_b: Some(&T6030_HWDATA_B), - registers: RegisterSet::G15S, - hwdata_words: &[], - globals_words: &[], -}; +pub(crate) static SOCS: [&Soc; 1] = [&T8122]; + +/// The table of the SoC with chip id `chip_id`, if the M3 runtime has one. +pub(crate) fn by_chip(chip_id: u32) -> Option<&'static Soc> { + SOCS.iter().copied().find(|soc| soc.chip_id == chip_id) +} /// T8122 (M3, G15G): one die, one cluster of ten core slots (eight or ten of them active). +/// +/// The compatibles, the mailbox and its interrupts are the T8122 device tree's. The identity is +/// the one the T8122 identity gate admits (`t8122_admission`: family 7, variant 2, revision 0x20, +/// core slots in the first core-mask word only), with the die count of the AGX3 identification +/// table (`hw::agx3::T8122`). The hardware configuration is `hw::t8122`; power configuration +/// comes from the boot loader. The rest of the runtime configuration is complete, but until the +/// runtime has passed on an M3 MacBook Air the SoC is refused, before the GPU is accessed, unless +/// `asahi.t8122_start=1` arms the start (`t8122_start`), which uses this table's values and adds a +/// performance-state cap. pub(crate) static T8122: Soc = Soc { name: "T8122", + gpu_name: "G15G", + chip_id: 0x8122, + gpu_variant: hw::GpuVariant::G, + gpu_revision: hw::GpuRevision::C0, + gpu_revision_id: hw::GpuRevisionID::C0, + board: "apple,t8122", + gpu: "apple,agx-t8122", + // The M3 MacBook Airs (13" J613, 15" J615). Allowed, not validated: the runtime has not run + // on either, and starts only behind a boot loader that hands the GPU over. + validated_boards: &[b"apple,j613", b"apple,j615"], + clusters: 1, + cores_per_cluster: 10, + mailbox_compatibles: &[ + b"apple,t8122-agx-asc-mailbox\0", + b"apple,t8122-asc-mailbox\0apple,asc-mailbox-v4\0", + ], + mailbox_interrupts: &[0, 723, 4, 0, 724, 4, 0, 725, 4, 0, 726, 4], + // The J613 ADT: sgx reg[0] (child 0x80000000 + arm-io 0x210000000) and gfx-asc reg[0]; the + // mailbox at asc + 0x8000 (t8122-gpu.dtsi). The same addresses as on T6030. + windows: Windows { + asc: 0x2_9240_0000, + sgx: 0x2_9000_0000, + mailbox: 0x2_9240_8000, + }, + id: IdWords { + version: 0x0702_2000, + version_mask: 0xffff_ff00, + // One die, one cluster per die; the other fields are not known. + counts: 0x0001_0100, + counts_mask: 0x000f_ff00, + }, + images: &crate::m3::m3_board::KNOWN_IMAGES_T8122, + firmware: Some(&crate::m3::m3_firmware::T8122_LAYOUT), + hwcfg: Some(&hw::t8122::HWCONFIG_T8122), + io_mappings: Some(&crate::m3::m3_adt_config::T8122_IO_MAPPINGS), + iomaps: Some(&crate::m3::m3_adt_config::T8122_IOMAPS), + // The boot loader's ladder from this machine's ADT (J613: eight voltages, up to 1338 MHz). + pstates: PstateTable::DeviceTree, + // The same SGX setup write as on T6030. + sgx_setup: Some((0xd14000, 0x70001)), hwdata_b: Some(&T8122_HWDATA_B), + // Neither is validated on G15G yet; userspace keeps its default ordering and visibility. + features: Features { + compute_wide_visibility: false, + fragment_dependency: false, + }, + power_from_boot_loader: true, + unported: &[ + "a pass on an M3 MacBook Air: until then the GPU starts only with asahi.t8122_start=1", + ], + // The performance state only. On the single-cluster G15G, reading the engine-busy or the + // fault-bank registers after every job hangs the SoC within seconds at about 1000 jobs/s; + // the performance-state read alone does not, and a per-frame readback saw no job retired + // before its writes were visible. + retire_mmio: 4, + // The fast-die controller's sensors (0x4248, both copies), and every sensor the MTR block + // enables as the alarm mask (sensors 3, 6, 8, 9, 11 and 14: 0x4b48, the upper half of the + // block's configuration word 0x4b480003). With the second fast-die copy at 0, or an alarm + // mask without sensor 8, the firmware stopped on its first MTR alarm. + mtr_masks: Some(MtrMasks { + fast_die: 0x4248, + alarm: 0x4b48, + }), + // The firmware's MTR alarm handler does a 64-bit atomic update of HwDataA +0x4350. The fixed + // allocation ends the 0x8a04-byte object at its page end, leaving HwDataA (+0x4580) only + // 4-byte aligned, and the update took an alignment fault. Starting the object 0x8a80 bytes + // before the same page end puts HwDataA on a 16-byte boundary. Aligned, the update then + // faulted as an unsupported atomic on the uncached mapping: map it cacheable. + hwdata_object: Some(HwDataObject { + address: T8122_HWDATA_ADDRESS, + cached: true, + }), registers: RegisterSet::G15G, + // On a T8122 with the 14.8.3 system firmware the shader clock runs at 3/4 of every requested + // state, so with the cap engaged the GPU stays at state 2 (462 MHz effective) under any load. + // With 0 it reaches the requested states (state 8: about 1000 MHz effective). + ut_engagement: 0, + // The boot loader's T8122 power model is a stand-in that overstates the power (its highest + // operating point is about 30 W). Until each Mac's fused leakage gives a real one, scale it + // to a 22 W target, about the GPU power budget of an M3 MacBook Air. + power_target_cap_mw: Some(22_000), hwdata_words: &T8122_HWDATA_WORDS, globals_words: &T8122_GLOBALS_WORDS, + // Exported on a J613 on every boot; used only with the boot loader's switch, which also makes + // value 1 its apple,core-leak-coef and the base of its operating points' powers. + leak_fuse: Some(LeakFuse { + values: c_str!("asahi,t8122-gpu-leak-fuse"), + switch: c_str!("asahi,t8122-gpu-fuse-leakage"), + }), + per_cluster_voltages: false, }; /// The T8122 HwData words that differ from the shared builder's (offsets into the object; @@ -104,7 +377,7 @@ pub(crate) static T8122: Soc = Soc { /// +0x1804, +0x1814 and +0x1860 and all-ones at +0x1848); /// - HwDataA +0x11e0: 30 (T6030: 40), a word of the shader-engine controller block; /// - HwDataA +0x1290: 125, and HwDataA +0x424c: 24000000, the 24 MHz reference clock. -static T8122_HWDATA_WORDS: [(usize, u32); 17] = [ +pub(crate) static T8122_HWDATA_WORDS: [(usize, u32); 17] = [ (0xa30, 0), (0xa34, 4), (0x17e0, 1), @@ -126,4 +399,18 @@ static T8122_HWDATA_WORDS: [(usize, u32); 17] = [ /// The T8122 Globals words that differ from the shared builder's: +0x9bc, the CDM backoff /// timeout (4) with the three bytes after it 0 (T6030: 1, 0, 0). -static T8122_GLOBALS_WORDS: [(usize, u32); 1] = [(0x9bc, 4)]; +pub(crate) static T8122_GLOBALS_WORDS: [(usize, u32); 1] = [(0x9bc, 4)]; + +/// The T8122 HwData object's firmware VA: 0x8a80 bytes before the end of the fixed allocation's +/// last page. +pub(crate) const T8122_HWDATA_ADDRESS: u64 = 0xffff_fc20_4070_4000 - 0x8a80; + +// The fixed HwData allocation (`m3_init_storage::allocation`): 0x8a04 bytes ending at the page +// end 0xfffffc2040704000. The moved object stays inside the same pages, and HwDataA is 16-byte +// aligned. +const _: () = { + let (fixed, size) = (0xffff_fc20_406f_b5fc_u64, 0x8a04_u64); + assert!(T8122_HWDATA_ADDRESS & !0x3fff == fixed & !0x3fff); + assert!(T8122_HWDATA_ADDRESS + size <= (fixed + size + 0x3fff) & !0x3fff); + assert!((T8122_HWDATA_ADDRESS + crate::m3::m3_adt_config::HWDATA_A as u64) % 16 == 0); +}; diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 9bd08d0257b0c..326e514808382 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Bounded M3 compute and graphics submissions through the common DRM scheduler/syncobjs. use kernel::{c_str,dma_fence::*,drm::sched,prelude::*,sync::{Arc,Mutex},new_mutex,xarray}; -use crate::{file,agx_uapi,mmu,queue,m3_drm::Shared}; +use crate::{file,m3::agx_uapi,mmu,queue,m3::m3_drm::Shared}; use core::sync::atomic::{AtomicBool, Ordering}; #[derive(Default)] pub(crate) struct Completion; @@ -32,14 +32,14 @@ impl Destination { #[derive(Copy,Clone)] pub(crate) enum Command { - Compute(crate::agx_compute::Control), + Compute(crate::m3::agx_compute::Control), Render{command:agx_uapi::UapiRenderCommand,usc:u64}, } pub(crate) struct Packet { pub(crate) vm:mmu::Vm, pub(crate) commands:KVec, timestamps:KVec<[Option;4]>, - pub(crate) attachments:KVec, + pub(crate) attachments:KVec, pub(crate) wide_visibility:KVec, completion:UserFence, finish_claimed: AtomicBool, @@ -47,7 +47,7 @@ pub(crate) struct Packet { pub(crate) urgency:u8, /// Who submitted the packet, for the failure log only. pub(crate) owner:Owner, - vm_job:Pin>>>, + vm_job:Pin>>>, } /// The process and thread that submitted a packet, recorded in the submit ioctl. Only logged: /// a renamed task leaves a stale name, which is harmless. @@ -57,8 +57,8 @@ impl Owner { fn current()->Self { let task=kernel::current!(); let leader=task.group_leader(); - Self {pid:leader.pid(),process:crate::m3_client::task_comm(leader), - thread:crate::m3_client::task_comm(task)} + Self {pid:leader.pid(),process:crate::m3::m3_client::task_comm(leader), + thread:crate::m3::m3_client::task_comm(task)} } fn name(comm:&[u8;16])->&str { let len=comm.iter().position(|&b|b==0).unwrap_or(comm.len()); @@ -84,18 +84,18 @@ impl Packet { // Claim error/signal ownership once; this flag is NOT proof that the // fence is signaled or that hardware DMA has stopped. if self.finish_claimed.swap(true, Ordering::Relaxed) { return; } - if let Err(e)=result {if let Ok(status) = self.vm.agx_status() { status.record(e.to_errno()); }self.completion.set_error(e);} + if let Err(e)=result {if let Some(status) = self.vm.status() { status.record(e); }self.completion.set_error(e);} // A scheduler job/fence and the runtime's last packet can outlive // hardware retirement. They must not keep the VM's active-job count // nonzero: GEM closes otherwise accumulate across completed frames. // On failure the runtime quarantines this guard with the packet until // firmware has provably stopped; signalling an error is not retirement. - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); let profile=crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming); - let before=if profile {crate::m3_runtime::physical_counter()} else {0}; + let before=if profile {crate::m3::m3_runtime::physical_counter()} else {0}; self.completion.signal(); if profile { - let after=crate::m3_runtime::physical_counter(); + let after=crate::m3::m3_runtime::physical_counter(); let status=match result {Ok(())=>0,Err(e)=>e.to_errno()}; pr_info!("M3_PACKET_FENCE vm={} packet={:#x} commands={} signal_before_tick={} signal_after_tick={} status={} counter_hz=24000000\n", self.vm.id(),self.profile_id(),self.commands.len(),before,after,status); @@ -113,7 +113,7 @@ impl sched::JobImpl for Job { fn run(job:&mut Self)->Result> { // Queue the packet and return its fence; the completion worker publishes and retires // it, so the scheduler can hand over further ready jobs meanwhile. - if let Err(e)=crate::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) { + if let Err(e)=crate::m3::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) { pr_err!("M3 scheduler: submission failed {:?}\n",e); // submit() failed before this packet could be published. job.packet.release_guard();job.packet.finish(Err(e)); @@ -122,9 +122,9 @@ impl sched::JobImpl for Job { } fn timed_out(job:&mut JobTimeout,_finished:bool)->sched::Status { // Retain the publishing branch's explicit diagnostic override. - if !crate::m3_params::timeout_nohang() { + if !crate::m3::m3_params::timeout_nohang() { if let Some(r)=Option::as_mut(&mut *job.shared.lock()) {r.health().mark_failed();} - crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); job.packet.finish(Err(ETIMEDOUT));return sched::Status::NoDevice; } let guard=job.shared.lock(); @@ -138,12 +138,12 @@ impl sched::JobImpl for Job { // device loss or corrupting an already signaled fence. if runtime.health().healthy() { return sched::Status::NoHang; } drop(guard); - crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); job.packet.finish(Err(EIO)); sched::Status::NoDevice } fn cancel(job:&mut Self) { - crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); job.packet.finish(Err(ECANCELED)); } } @@ -178,9 +178,9 @@ impl Queue { } } impl queue::Queue for Queue { - fn submit(&mut self,_id:u64,mut syncs:KVec,in_sync_count:usize, + fn submit(&mut self,_reg_data:&crate::driver::AsahiRegData<'_>,_id:u64,mut syncs:KVec,in_sync_count:usize, raw:&[u8],objects:Pin<&xarray::XArray>>)->Result { - let vm_job=self.vm.retain_m3_job()?; + let vm_job=self.vm.retain_job()?; let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new(); while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? { if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command { @@ -195,7 +195,7 @@ impl queue::Queue for Queue { Destination::resolve(objects,payload.vertex_timestamps.end)?, Destination::resolve(objects,payload.fragment_timestamps.start)?, Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?; - flushes.push(crate::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; + flushes.push(crate::m3::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; visibility.push(false,GFP_KERNEL)?; continue; } @@ -217,10 +217,10 @@ impl queue::Queue for Queue { pr_err!("M3 compute translation rejected {:?}: stream={:#x}..{:#x} helper={:?}\n",e,payload.control_stream_base,payload.control_stream_end,payload.helper); EINVAL })?; - crate::m3_client::check_compute_launches(&self.vm,&AddressSpace(&self.vm),c.control_stream_base,c.control_stream_end)?; + crate::m3::m3_client::check_compute_launches(&self.vm,&AddressSpace(&self.vm),c.control_stream_base,c.control_stream_end)?; timestamps.push([Destination::resolve(objects,payload.timestamps.start)?, Destination::resolve(objects,payload.timestamps.end)?,None,None],GFP_KERNEL)?; - commands.push(Command::Compute(crate::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; + commands.push(Command::Compute(crate::m3::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; } parser.finish().map_err(|_|EINVAL)?; let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, @@ -238,8 +238,8 @@ impl queue::Queue for Queue { } fn encode_attachments(vm: &mmu::Vm, list: &agx_uapi::UapiAttachmentList) - -> Result { - use crate::agx_attachments::{Attachments, CAPACITY}; + -> Result { + use crate::m3::agx_attachments::{Attachments, CAPACITY}; use agx_uapi::{GpuAddressSpace, GpuAccess}; if list.as_slice().len()>CAPACITY {return Err(E2BIG);} let mut ranges = [(0, 0); CAPACITY]; @@ -262,7 +262,7 @@ fn validate_render(r: agx_uapi::UapiRenderCommand, usc: u64, space: &AddressSpac if r.sample_size % 8 != 0 {return Err(ENOTSUPP);} let empty = agx_uapi::UapiHelperProgram { binary:0, config:0, data:0 }; // Early tiling (render flag bit 5) only where it is advertised. - let stage_scope=if early_tiling && *crate::module_parameters::m3_early_tiling.value()!=0 {1<<5} else {0}; + let stage_scope=if early_tiling && crate::m3::m3_params::early_tiling() {1<<5} else {0}; if r.flags & !(2|16|(1<<18)|stage_scope) != 0 || !matches!(r.samples, 1 | 2 | 4) || r.sampler_count != 0 || r.sampler_heap != 0 || r.vertex_helper != empty || r.fragment_helper != empty @@ -279,7 +279,7 @@ fn validate_render(r: agx_uapi::UapiRenderCommand, usc: u64, space: &AddressSpac || u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)>32768 { return Err(EINVAL); } - crate::agx_render_state::compact(r.vdm_base).map_err(|_| EINVAL)?; + crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_| EINVAL)?; if r.vdm_base & 3 != 0 || !space.covers(r.vdm_base,4,GpuAccess::Read) || r.scissor_base == 0 || r.scissor_base & 7 != 0 || !space.covers(r.scissor_base,8,GpuAccess::Read) { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/m3/m3_thermal.rs b/drivers/gpu/drm/asahi/m3/m3_thermal.rs index 59a72187c9699..e30f8fe011332 100644 --- a/drivers/gpu/drm/asahi/m3/m3_thermal.rs +++ b/drivers/gpu/drm/asahi/m3/m3_thermal.rs @@ -47,7 +47,7 @@ use core::sync::atomic::{AtomicBool, Ordering}; use kernel::{ - bindings, c_str, device, + c_str, device, error::to_result, new_spinlock, prelude::*, @@ -58,9 +58,9 @@ use kernel::{ }; use crate::{ - m3_adt_config::PstatePolicy, - m3_params::{self, ThermalMode}, - m3_thermal_policy::{Change, Policy, Reading, Source, Why, S}, + m3::m3_adt_config::PstatePolicy, + m3::m3_params::{self, ThermalMode}, + m3::m3_thermal_policy::{Change, Policy, Reading, Source, Why, S}, }; /// The thermal zones read, each the hottest of a set of SoC die sensors: the SMC's die sensors @@ -78,7 +78,7 @@ fn required_zones() -> u32 { let Ok(compatible) = root.get_property::>(c_str!("compatible")) else { return J516_REQUIRED_ZONES; }; - crate::m3_thermal_policy::required_zones(&compatible) + crate::m3::m3_thermal_policy::required_zones(&compatible) } /// Sensor polling period while the GPU is busy and while it is idle, in ms. @@ -122,13 +122,18 @@ impl_has_delayed_work! { impl HasDelayedWork for Sensor { self.work } } +extern "C" { + // include/linux/thermal.h; not in 7.2's generated bindings. + fn thermal_zone_get_temp_by_name(name: *const core::ffi::c_char, temp: *mut i32) -> core::ffi::c_int; +} + /// One reading of the zone `name`, in millidegrees Celsius; ENODEV if there is no such zone. fn read_zone(name: &CStr) -> Result { let mut temp: i32 = 0; // SAFETY: name is NUL-terminated and temp is a valid output pointer. The // thermal core acquires a device reference under its registry lock and // keeps it through sampling, so zone unregistration cannot free it here. - to_result(unsafe { bindings::thermal_zone_get_temp_by_name(name.as_char_ptr(), &mut temp) })?; + to_result(unsafe { thermal_zone_get_temp_by_name(name.as_char_ptr(), &mut temp) })?; Ok(temp) } diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index 3d900a80fef38..1d10657c02f66 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -43,6 +43,96 @@ pub(crate) fn refuse_probe(pdev: &platform::Device>, chip: Chip) -> Err ENODEV } +/// Whether the M3 runtime may start on this T8122 (M3, G15G), whose GPU the boot loader handed +/// over: its identity gate passes (only DT properties are read) and the backend selection allows +/// the board. The runtime then admits the boot loader's description, and refuses the SoC unless +/// `asahi.t8122_start=1` arms the start ([`t8122_start`]). +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +pub(crate) fn t8122_admitted(pdev: &platform::Device>) -> Result<&'static m3_soc::Soc> { + let identity = t8122_admission::read_identity(pdev).map_err(|error| { + dev_info!( + pdev.as_ref(), + "G15G: T8122 identity or ABI rejected ({:?}); startup refused\n", + error + ); + ENODEV + })?; + dev_info!( + pdev.as_ref(), + "G15G: T8122 revision 2.0 identity admitted ({} active / {} slots)\n", + identity.active_cores, + identity.core_slots + ); + if !m3_params::t8122_backend(pdev) { + return Err(ENODEV); + } + Ok(&m3_soc::T8122) +} + +// The T8122 runtime, ported from the 7.1.12 release-20 kernel (c46d7dcdb871). +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_attachments; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_compute; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_memory; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_resources; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_status; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_uapi; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod g15_raw; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod hw; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod identity; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod initdata; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_adt_config; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_board; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_client; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_completion; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_compute; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_config; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_device; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +pub(crate) mod m3_drm; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_firmware; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_memory; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_params; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_pass; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_pool; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_render; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_resources; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_rtkit; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_runtime; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_submit; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_thermal; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod t8122_knobs; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod t8122_start; + mod m3_compute_layout; mod m3_compute_sequence; @@ -103,6 +193,8 @@ mod m3_fragment_command; mod t8122_admission; -mod m3_soc; +// The runtime's SoC tables; they name the runtime's own modules. +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +pub(crate) mod m3_soc; mod t8122_hwdata; diff --git a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs index f9c3ff08006d5..b48e561f677ef 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs @@ -172,7 +172,7 @@ fn given(param: u64, default: u64) -> Option { /// Resolve the parameters. The first parameter with a value it does not accept is the refusal. pub(crate) fn resolve(raw: &Raw) -> Result { let refuse = |name, accepts| Refusal { name, accepts }; - let initdata_version = given(raw.initdata_version, crate::m3_firmware::G15_V14_8_3_INITDATA) + let initdata_version = given(raw.initdata_version, crate::m3::m3_firmware::G15_V14_8_3_INITDATA) .filter(|v| *v != 0) .ok_or(refuse("t8122_initdata_version", "a nonzero number, at most 0xfffffffffffffffd"))?; let fender = match given(raw.fender, FENDER_RULE as u64) { diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs index cf0ee658e8da5..ca9f21b56dbb6 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_start.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs @@ -27,13 +27,13 @@ use core::mem::offset_of; use kernel::{device, prelude::*}; use crate::{ - fw::initdata::raw, - initdata::G15RuntimeHwDataB, - m3_adt_config::T8122_IO_MAPPINGS, - m3_init_storage::{self as storage, IoMap}, - m3_soc::{IoMapping, Soc, T8122_HWDATA_B}, + m3::g15_raw::raw, + m3::initdata::G15RuntimeHwDataB, + m3::m3_adt_config::T8122_IO_MAPPINGS, + m3::m3_init_storage::{self as storage, IoMap}, + m3::m3_soc::{IoMapping, Soc, T8122_HWDATA_B}, pgtable::{prot, Prot}, - t8122_knobs::{ + m3::t8122_knobs::{ self as knobs, ClockGen, Start, Values, CLOCK_GEN_E1C, CLOCK_GEN_E5C, FENDER_ADT, FENDER_RULE, }, @@ -140,7 +140,7 @@ const _: () = { assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_b) == 0x17c8); // An armed start skips `require_complete`. The only `unported` item of the T8122 table is the // opt-in itself; if another is added, this fails the build until the start accounts for it. - assert!(crate::m3_soc::T8122.unported.len() == 1); + assert!(crate::m3::m3_soc::T8122.unported.len() == 1); }; /// The values of one armed boot. @@ -149,7 +149,7 @@ pub(crate) struct Experiment(Values); /// Whether `soc` is the T8122 table. pub(crate) fn is_t8122(soc: &Soc) -> bool { - core::ptr::eq(soc, &crate::m3_soc::T8122) + core::ptr::eq(soc, &crate::m3::m3_soc::T8122) } impl Experiment { @@ -182,7 +182,7 @@ impl Experiment { /// they are the T8122 table's own IO maps. pub(crate) fn iomaps(&self) -> &'static [IoMap] { if self.0.fender == FENDER_RULE && self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C) { - return &crate::m3_adt_config::T8122_IOMAPS; + return &crate::m3::m3_adt_config::T8122_IOMAPS; } &IOMAPS[self.variant()][..self.mapping_count()] } @@ -245,7 +245,7 @@ pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> if !is_t8122(soc) { return Ok(None); } - let raw = crate::m3_params::t8122_params(); + let raw = crate::m3::m3_params::t8122_params(); match knobs::start(&raw) { Start::On => {} Start::Off => { @@ -354,7 +354,7 @@ pub(crate) fn refused(dev: &device::Device, experiment: Option<&Experiment>, sta /// Log, on a T8122 armed with `asahi.t8122_start=1`, that resource admission refused it. pub(crate) fn not_admitted(dev: &device::Device, soc: &Soc) { - if is_t8122(soc) && knobs::start(&crate::m3_params::t8122_params()) == Start::On { + if is_t8122(soc) && knobs::start(&crate::m3::m3_params::t8122_params()) == Start::On { dev_err!( dev, "M3 G15G start: not armed: resource admission refused the GPU (see the lines above); the boot loader did not hand it over completely\n" diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index e868e30baca4c..dc828a460202d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -117,6 +117,8 @@ mod bind; mod context; mod shared; mod lifetime; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3; mod residency; pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; pub(crate) use bind::PreparedUserBindBatch; diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs new file mode 100644 index 0000000000000..a63101a813426 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The VM and binding helpers of the M3 T8122 runtime (`crate::m3`), which 7.1.12's `mmu.rs` +//! provided for it. + +use super::*; + +impl Vm { + /// Whether `other` is the same DRM VM. + pub(crate) fn same(&self, other: &Vm) -> bool { + self.id == other.id + } + + /// Copy the GPU-readable bytes this VM maps at `iova`, through the point of coherency + /// ([`Vm::read_user`]). M3 compute validates the client control stream before publishing it. + pub(crate) fn read_bytes(&self, iova: u64, out: &mut [u8]) -> Result { + if out.is_empty() { + return Err(EINVAL); + } + self.read_user(iova, out) + } + + /// Snapshot the first `out.len()` TTBAT entries as `(ttb0, ttb1)` pairs. The TTBAT is + /// ordinary memory, so this reads no GPU register and is safe with the cores gated. + pub(crate) fn context_roots(&self, out: &mut [(u64, u64)]) -> Result { + let inner = lock_vm(&self.gpuvm, None, false)?; + let shared = inner.uat_inner.lock(); + let ttbs = shared.ttbs(); + for (slot, entry) in out.iter_mut().enumerate().take(UAT_NUM_CTX) { + *entry = ( + ttbs[slot].ttb0.load(Ordering::Acquire), + ttbs[slot].ttb1.load(Ordering::Acquire), + ); + } + Ok(()) + } + + /// Map `size` bytes of MMIO or DMA memory at `phys` into this Vm, at `iova`, or at a free + /// address in `range` when `iova` is `None`. Unlike [`Vm::map_io`] this logs nothing on + /// success: the runtime maps job views on every submission. + fn map_io_m3_inner( + &self, + iova: Option, + range: Range, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + if size == 0 || (iova.unwrap_or(0) as usize | phys | size) & UAT_PGMSK != 0 { + pr_err!( + "MMU: M3 KernelMapping {:#x}:{:#x} -> {:#x?} is not page-aligned\n", + phys, + size, + iova + ); + return Err(EINVAL); + } + // SAFETY: As in `map_io`, the exec lock is only used for this mapping. + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + let inner = vm_exec.unique_gpuvm.data(); + let payload = KernelMappingInner { + owner: self.gpuvm.clone(), + uat_inner: inner.uat_inner.clone(), + prot, + bo: None, + _gem: None, + offset: 0, + mapped_size: size, + }; + let node = match iova { + Some(iova) => inner.mm.reserve_node(payload, iova, size as u64, 0)?, + None => inner.mm.insert_node_in_range( + payload, + size as u64, + UAT_PGSZ as u64, + 0, + range.start, + range.end, + mm::InsertMode::Best, + )?, + }; + let iova = node.start(); + let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; + inner.page_table.prepare_map(iova..end)?; + let mutation = inner.mapping_mutation(); + let ret = inner + .page_table + .map_pages(iova..end, phys as PhysicalAddr, prot, false); + if ret.is_err() { + inner.rollback_failed_map(iova, size as u64); + } + // Drop the exec lock before the node, as in `map_io`. + drop(mutation); + let _ = inner; + ret?; + Ok(KernelMapping(node)) + } + + /// [`Vm::map_io`] at `iova`, without the per-mapping log line. + pub(crate) fn map_io_m3( + &self, + iova: u64, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + self.map_io_m3_inner(Some(iova), 0..0, phys, size, prot) + } + + /// [`Vm::map_io`] at a free address in `range`. + pub(crate) fn map_io_in_range( + &self, + range: Range, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + self.map_io_m3_inner(None, range, phys, size, prot) + } +} + +impl VmBind { + /// The bound `Vm` itself, for diagnostics that need to walk its tables. + pub(crate) fn vm(&self) -> &Vm { + &self.0 + } + + /// Whether this retained binding belongs to the supplied DRM VM. + pub(crate) fn matches(&self, vm: &Vm) -> bool { + self.0.id == vm.id + } + + /// Physical base of the bound VM's page-table root. + pub(crate) fn root(&self) -> u64 { + self.0.ttb() + } + + /// The bound VM's driver-assigned id. + pub(crate) fn vm_id(&self) -> u64 { + self.0.id + } +} diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 5f7e572a3a462..87ec3bd73658b 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -318,6 +318,18 @@ impl RtKit { }) } + /// Returns whether the coprocessor is running (not crashed, IOP and AP power states on). + pub fn is_running(&self) -> bool { + // SAFETY: `rtk` is valid per the type invariant. The call only reads state. + unsafe { bindings::apple_rtkit_is_running(self.rtk) } + } + + /// Returns whether the coprocessor has reported a crash. + pub fn is_crashed(&self) -> bool { + // SAFETY: `rtk` is valid per the type invariant. The call only reads state. + unsafe { bindings::apple_rtkit_is_crashed(self.rtk) } + } + /// Checks if an endpoint is present pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs index 35ff049ff0783..095a51627edfe 100644 --- a/rust/kernel/sync/completion.rs +++ b/rust/kernel/sync/completion.rs @@ -90,6 +90,28 @@ impl Completion { self.inner.get() } + /// Publish one completion token, including from interrupt context. + #[inline] + pub fn complete(&self) { + // SAFETY: The pinned completion remains initialized for this call. + unsafe { bindings::complete(self.as_raw()) }; + } + + /// Consume one pending token without blocking. + #[inline] + pub fn try_wait_for_completion(&self) -> bool { + // SAFETY: The pinned completion remains initialized for this call. + unsafe { bindings::try_wait_for_completion(self.as_raw()) } + } + + /// Wait for a token for at most `timeout` jiffies; return whether one arrived. + #[inline] + pub fn wait_for_completion_timeout(&self, timeout: crate::time::Jiffies) -> bool { + let timeout = timeout.min(core::ffi::c_long::MAX as _); + // SAFETY: The completion remains live while this method waits. + unsafe { bindings::wait_for_completion_timeout(self.as_raw(), timeout as _) != 0 } + } + /// Signal all tasks waiting on this completion. /// /// This method wakes up all tasks waiting on this completion; after this operation the From e16c5bbae3f8f6cd101fc4890e040c50f1aeeee8 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:52 -0400 Subject: [PATCH 04/37] drm/asahi: m3: map each VM's dummy page where the firmware expects it 7.1.12 mapped each VM's dummy page at the backend's unknown_page(), which the M3 runtime returned as 0x2ff_ffff_8000, the address its InitData gives the firmware (unk_30). On 7.2 file.rs maps it at uat_geometry().unk_page() instead, which is 0x3ff_ffff_8000 with the runtime's 42-bit geometry, so the firmware's unknown page was unmapped in every T8122 VM. Add a gpu::Gpu::unknown_page() hook, defaulting to the geometry's page so no other GPU changes, and have the M3 backend return RUNTIME_UNKNOWN_PAGE, which also ends its user range, so the two can't drift again. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/file.rs | 2 +- drivers/gpu/drm/asahi/gpu.rs | 4 ++++ drivers/gpu/drm/asahi/m3/initdata.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_drm.rs | 9 ++++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 242f6cdcd3f83..f2f055931d29a 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -549,7 +549,7 @@ impl File { } let dummy_mapping = dummy_obj.map_at( &vm, - gpu.uat_geometry().unk_page(), + gpu.unknown_page(), mmu::PROT_GPU_SHARED_RW, true, )?; diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index b9283efb470a9..a06b5da5ed6b1 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -344,6 +344,10 @@ pub(crate) trait Gpu: Send + Sync { fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { Ok(()) } + /// GPU VA of each VM's dummy page, which must match the address the firmware was given. + fn unknown_page(&self) -> u64 { + self.uat_geometry().unk_page() + } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; /// Create a new user command queue. diff --git a/drivers/gpu/drm/asahi/m3/initdata.rs b/drivers/gpu/drm/asahi/m3/initdata.rs index 2348d075a731c..ab0e1643468f7 100644 --- a/drivers/gpu/drm/asahi/m3/initdata.rs +++ b/drivers/gpu/drm/asahi/m3/initdata.rs @@ -122,7 +122,7 @@ impl G15Options { /// Start of the USC (shader code) window in the runtime backend's GPU VA layout. const RUNTIME_USC_BASE: u64 = 0x10_0000_0000; /// The runtime backend's unknown-page address, the end of its 42-bit user VA range. -const RUNTIME_UNKNOWN_PAGE: u64 = 0x2ff_ffff_8000; +pub(crate) const RUNTIME_UNKNOWN_PAGE: u64 = 0x2ff_ffff_8000; /// Maximum number of entries of a firmware performance-state table, including the off state. const PSTATE_TABLE_ENTRIES: usize = 16; diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 2d93823e67d01..dc8f1a074e809 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -232,7 +232,14 @@ impl gpu::Gpu for Backend { value as u32 } fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } - fn user_range(&self) -> Range { mmu::UAT_PGSZ as u64..0x2ff_ffff_8000 } + fn user_range(&self) -> Range { + mmu::UAT_PGSZ as u64..crate::m3::initdata::RUNTIME_UNKNOWN_PAGE + } + /// The dummy page sits where InitData tells the firmware it is (unk_30), as on 7.1.12; the + /// 42-bit geometry's own unk_page() is a different address. + fn unknown_page(&self) -> u64 { + crate::m3::initdata::RUNTIME_UNKNOWN_PAGE + } fn tracked_submission(&self) -> bool { true } /// Only M3-aware userspace may create a VM (7.1.12's per-file client gate). fn admit_vm(&self, dev: &kernel::device::Device, params_size: u64, flags: u32) -> Result { From eae5bfa6b7d34549e157bb3ec77276c3ca6e7868 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:53 -0400 Subject: [PATCH 05/37] drm/asahi: m3: quarantine a failed job's VM before signalling its fence Packet::finish signalled an error fence while the packet could still hold its VM job guard, and nothing on the M3 path quarantined it, so after a timeout userspace could unmap and close memory that work the firmware had not provably stopped was using. Unmap invalidates the TLB, so this ended in a GPU fault rather than freed memory, as on 7.1.12, but 7.2's shared job lifetime can block it. As G15 does, quarantine a guard that is still held before the error fence becomes visible: mapping changes wait until the runtime drops the packet. Guards already released (work never published, or retired) are unaffected. Fix the comment that said this already happened. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_submit.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 326e514808382..ca16e18f10fbf 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -84,12 +84,19 @@ impl Packet { // Claim error/signal ownership once; this flag is NOT proof that the // fence is signaled or that hardware DMA has stopped. if self.finish_claimed.swap(true, Ordering::Relaxed) { return; } - if let Err(e)=result {if let Some(status) = self.vm.status() { status.record(e); }self.completion.set_error(e);} + if let Err(e)=result { + // A guard still held here belongs to work the firmware may still run: block + // mapping changes before the error fence is visible (as G15 does), so userspace + // can't unmap memory under it. It stays with the packet until the runtime drops + // the packet, after the firmware has provably stopped. + if let Some(guard)=self.vm_job.lock().as_ref() {guard.quarantine();} + if let Some(status) = self.vm.status() { status.record(e); } + self.completion.set_error(e); + } // A scheduler job/fence and the runtime's last packet can outlive // hardware retirement. They must not keep the VM's active-job count // nonzero: GEM closes otherwise accumulate across completed frames. - // On failure the runtime quarantines this guard with the packet until - // firmware has provably stopped; signalling an error is not retirement. + // Signalling an error is not retirement. crate::m3::agx_memory::publish(); let profile=crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming); let before=if profile {crate::m3::m3_runtime::physical_counter()} else {0}; From c9dcc611f3f1646e49a5451a9004a7744b7d02f0 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:53 -0400 Subject: [PATCH 06/37] drm/asahi: m3: keep the adopted UAT until ASC stops on a failed start Once adopt_uat() points TTBAT context 0 at the kernel-owned lower root, a failure in Config::new() or the window sync dropped the UAT and its window inside the start closure, before stop_asc(), and the branch that retains everything when ASC fails to stop could not retain tables that were already freed. Keep both outside the closure: drop them only after ASC has stopped, and leak them with the rest when it does not. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index b12fc73f9f248..28bbea3c86889 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -997,6 +997,9 @@ impl Runtime { } // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. let mut started=false; + // The adopted UAT and its window live outside the closure: once TTBAT context 0 names + // the kernel-owned lower root, a failure must not free those tables before ASC stops. + let mut adopted=None; let prepared=(|| -> Result<_> { device.start_asc(pdev)?; Pin::new(&mut transport).wake()?; @@ -1010,23 +1013,28 @@ impl Runtime { // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as // every context's upper half, and the host window (root entry 2) through the // shared L2. The RTKit is awake and no initdata or GPU job has been published. - let (uat,mut window)=adopt_uat(&drm,device.soc(),&device.firmware().resources)?; - let config=Config::new(&drm,&uat,device.firmware(),&contents)?; + let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources)?); + let config=Config::new(&drm,uat,device.firmware(),&contents)?; // Link every kernel table the configuration created before the firmware sees an // address in it, then drop any translation cached before they existed. - window.sync_all(&uat)?; + window.sync_all(uat)?; crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); - Ok((uat,window,config)) + Ok(config) })(); - let (uat,window,config)=match prepared { - Ok(v)=>v, - Err(e)=>{ + let (uat,window,config)=match (prepared,adopted.take()) { + (Ok(config),Some((uat,window)))=>(uat,window,config), + (prepared,adopted)=>{ + let e=prepared.err().unwrap_or(EINVAL); crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; core::mem::forget(transport);core::mem::forget(state); + core::mem::forget(adopted); core::mem::forget(drm);core::mem::forget(device); + } else { + // ASC has stopped: the firmware no longer walks the adopted tables. + drop(adopted); } return Err(e); } From 34b754f9d5cf09417d043ed1c77aafdf03041ad2 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:54 -0400 Subject: [PATCH 07/37] drm/asahi: m3: don't let the compute stream debug read fail a job Compute::new() read 48 bytes of the client's CDM stream for a debug message, whatever the stream's length, and failed the job if the read did. A short stream ending at a mapped page boundary with the next page unmapped passed the stream checks but then failed its first submission, even with debug logging off (as on 7.1.12). Read no further than the stream, and only log when the read succeeds. Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_compute.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index eab8bf71f7bcd..34aa0c0d2cd3f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -96,9 +96,12 @@ impl Compute { objects[storage::COMMAND].gpu_va()?,control.base,control.end,objects[storage::POOL].va()); // The M3 compute stream uses absolute IOTO addresses and does not // include the USC_EXEC_BASE_CP register. + // Diagnostic only: read no further than the stream, and never fail the job over it. let mut stream=[0u8;48]; - vm.read_bytes(control.base,&mut stream)?; - crate::cls_dev_dbg!(Compute,dev,"M3 client CDM bytes={:02x?}\n",stream); + let len=core::cmp::min(control.end.saturating_sub(control.base),stream.len() as u64) as usize; + if vm.read_bytes(control.base,&mut stream[..len]).is_ok() { + crate::cls_dev_dbg!(Compute,dev,"M3 client CDM bytes={:02x?}\n",&stream[..len]); + } agx_memory::publish(); // Finish the write-combining page-table stores before broadcasting // the invalidation. From 1694a19aa4e7eb37fc2ff58b9e7c3dae20e6a6e5 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:24:02 -0500 Subject: [PATCH 08/37] Prepare the original context for the carried M3 timestamp dependency The following original G17 sync commit includes the M3 timestamp adapter. Restore current Aurora integration in the separate adaptation afterwards. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/asahi_core.rs | 100 +- drivers/gpu/drm/asahi/driver.rs | 259 +-- drivers/gpu/drm/asahi/file.rs | 273 +-- drivers/gpu/drm/asahi/g15/command.rs | 40 +- drivers/gpu/drm/asahi/g15/mod.rs | 69 +- drivers/gpu/drm/asahi/g15/uat.rs | 14 +- drivers/gpu/drm/asahi/g17/buffer.rs | 80 +- drivers/gpu/drm/asahi/g17/buffer/target.rs | 137 +- drivers/gpu/drm/asahi/g17/capacity.rs | 93 - drivers/gpu/drm/asahi/g17/command.rs | 50 +- drivers/gpu/drm/asahi/g17/completion.rs | 330 +--- drivers/gpu/drm/asahi/g17/context.rs | 245 +-- drivers/gpu/drm/asahi/g17/context/identity.rs | 265 +-- drivers/gpu/drm/asahi/g17/data_segment.rs | 56 - drivers/gpu/drm/asahi/g17/dependency.rs | 132 -- drivers/gpu/drm/asahi/g17/event.rs | 16 +- drivers/gpu/drm/asahi/g17/feed.rs | 98 +- drivers/gpu/drm/asahi/g17/freelist.rs | 178 +- drivers/gpu/drm/asahi/g17/fw/buffer.rs | 4 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 64 +- drivers/gpu/drm/asahi/g17/fw/compute.rs | 85 +- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 157 +- .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 619 ------ drivers/gpu/drm/asahi/g17/fw/kick.rs | 30 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 5 +- drivers/gpu/drm/asahi/g17/fw/queue.rs | 137 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 377 +--- drivers/gpu/drm/asahi/g17/fw/variant.rs | 140 -- drivers/gpu/drm/asahi/g17/initdata.rs | 489 ++--- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 230 --- drivers/gpu/drm/asahi/g17/job.rs | 143 +- drivers/gpu/drm/asahi/g17/kick.rs | 459 +---- drivers/gpu/drm/asahi/g17/mod.rs | 838 ++------ drivers/gpu/drm/asahi/g17/object.rs | 208 +- drivers/gpu/drm/asahi/g17/power.rs | 205 +- drivers/gpu/drm/asahi/g17/preparation.rs | 249 +-- drivers/gpu/drm/asahi/g17/queue/admission.rs | 821 +------- drivers/gpu/drm/asahi/g17/queue/compute.rs | 1238 ++---------- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 32 +- .../drm/asahi/g17/queue/compute/retirement.rs | 623 +----- drivers/gpu/drm/asahi/g17/queue/mod.rs | 209 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 1087 ++--------- .../gpu/drm/asahi/g17/queue/render/memory.rs | 140 +- .../gpu/drm/asahi/g17/queue/render/parked.rs | 44 +- .../drm/asahi/g17/queue/render/publication.rs | 42 +- .../drm/asahi/g17/queue/render/retirement.rs | 189 +- .../gpu/drm/asahi/g17/queue/render/warm.rs | 3 +- drivers/gpu/drm/asahi/g17/recovery.rs | 229 +-- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 131 -- drivers/gpu/drm/asahi/g17/recovery/slot.rs | 8 +- drivers/gpu/drm/asahi/g17/replay.rs | 170 -- drivers/gpu/drm/asahi/g17/runtime.rs | 1684 ++--------------- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 196 +- .../drm/asahi/g17/runtime/compute_control.rs | 31 +- .../drm/asahi/g17/runtime/compute_replay.rs | 26 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 164 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 460 +---- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 13 +- .../drm/asahi/g17/runtime/render/observe.rs | 118 +- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 46 +- .../drm/asahi/g17/runtime/render/prepare.rs | 87 +- .../drm/asahi/g17/runtime/render/replay.rs | 148 +- .../drm/asahi/g17/runtime/render/settle.rs | 30 +- .../drm/asahi/g17/runtime/render/submit.rs | 572 +----- .../drm/asahi/g17/runtime/render/teardown.rs | 61 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 25 +- drivers/gpu/drm/asahi/g17/teardown.rs | 163 -- drivers/gpu/drm/asahi/g17/timeout.rs | 326 +--- drivers/gpu/drm/asahi/g17/validation.rs | 179 +- drivers/gpu/drm/asahi/gem.rs | 101 +- drivers/gpu/drm/asahi/gpu.rs | 123 +- drivers/gpu/drm/asahi/hw/mod.rs | 2 - drivers/gpu/drm/asahi/hw/t6040.rs | 216 --- drivers/gpu/drm/asahi/hw/t8140.rs | 41 +- drivers/gpu/drm/asahi/m3/m3_submit.rs | 9 +- drivers/gpu/drm/asahi/mem.rs | 48 +- drivers/gpu/drm/asahi/memacct.rs | 549 ------ drivers/gpu/drm/asahi/mmu.rs | 1259 +++--------- drivers/gpu/drm/asahi/mmu/bind.rs | 17 +- drivers/gpu/drm/asahi/mmu/lifetime.rs | 79 +- drivers/gpu/drm/asahi/mmu/ppl.rs | 71 - drivers/gpu/drm/asahi/mmu/residency.rs | 71 +- drivers/gpu/drm/asahi/mmu/shared.rs | 4 +- drivers/gpu/drm/asahi/object.rs | 42 +- drivers/gpu/drm/asahi/pgtable.rs | 449 ++--- drivers/gpu/drm/asahi/queue/common.rs | 2 +- drivers/gpu/drm/asahi/queue/mod.rs | 84 +- drivers/gpu/drm/asahi/regs.rs | 75 +- drivers/gpu/drm/asahi/sysfs.c | 196 +- drivers/gpu/drm/asahi/sysfs.h | 23 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 96 +- drivers/gpu/drm/asahi/timestamp.rs | 220 --- drivers/gpu/drm/asahi/util.rs | 11 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 46 +- include/drm/drm_gem_shmem_helper.h | 15 +- include/uapi/drm/asahi_drm.h | 127 +- rust/bindings/bindings_helper.h | 24 - rust/helpers/mm.c | 12 - rust/kernel/drm/device.rs | 27 +- rust/kernel/drm/file.rs | 86 - rust/kernel/drm/gem/shmem.rs | 487 ++--- rust/kernel/mm.rs | 53 - 102 files changed, 2606 insertions(+), 18228 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/g17/capacity.rs delete mode 100644 drivers/gpu/drm/asahi/g17/data_segment.rs delete mode 100644 drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs delete mode 100644 drivers/gpu/drm/asahi/g17/fw/variant.rs delete mode 100644 drivers/gpu/drm/asahi/g17/initdata/v25g83.rs delete mode 100644 drivers/gpu/drm/asahi/g17/recovery/safety.rs delete mode 100644 drivers/gpu/drm/asahi/g17/replay.rs delete mode 100644 drivers/gpu/drm/asahi/hw/t6040.rs delete mode 100644 drivers/gpu/drm/asahi/memacct.rs delete mode 100644 drivers/gpu/drm/asahi/mmu/ppl.rs delete mode 100644 drivers/gpu/drm/asahi/timestamp.rs diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index decf16ea09647..0fbe6e728773d 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -4,7 +4,6 @@ //! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. mod alloc; -mod binding; mod buffer; mod channel; mod cleanup; @@ -14,20 +13,19 @@ mod event; mod file; mod float; mod fw; -mod fwid; +mod gem; +mod g17; #[cfg(CONFIG_DRM_ASAHI_G15)] mod g15; -mod g17; -mod gem; - -#[cfg(CONFIG_DRM_ASAHI_G16)] -mod g16; +// Without the G15 backend, only the UAT adoption the T8122 runtime shares with it. +#[cfg(all(CONFIG_DRM_ASAHI_UAT_ADOPTION, not(CONFIG_DRM_ASAHI_G15)))] +#[path = "g15/adopt.rs"] +mod g15; mod gpu; mod hw; mod initdata; -mod m3; mod mem; -mod memacct; +mod m3; mod microseq; mod mmu; mod object; @@ -36,10 +34,7 @@ mod queue; mod regs; mod slotalloc; mod stats; -mod stats_raw; mod sysfs_exports; -mod timestamp; -mod usage; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -51,33 +46,12 @@ use kernel::prelude::*; type G15Registration = kernel::driver::Registration>>; -#[cfg(CONFIG_DRM_ASAHI_G16)] -type G16Registration = kernel::driver::Registration>; -#[cfg(not(CONFIG_DRM_ASAHI_G16))] -type G16Registration = (); - -fn register_g16(module: &'static kernel::ThisModule) -> Result>>> { - #[cfg(CONFIG_DRM_ASAHI_G16)] - { - Ok(Some(KBox::pin_init( - kernel::driver::Registration::new(g16::DRIVER_NAME, module), - GFP_KERNEL, - )?)) - } - #[cfg(not(CONFIG_DRM_ASAHI_G16))] - { - let _ = module; - Ok(None) - } -} - #[pin_data] struct DriverModule { #[cfg(CONFIG_DRM_ASAHI_G15)] #[pin] _g15: G15Registration, // Field drop order unregisters/joins driver producers before queue teardown. - _g16: Option>>, #[pin] _driver: kernel::driver::Registration>, _cleanup: cleanup::QueueOwner, @@ -87,18 +61,15 @@ struct DriverModule { impl kernel::InPlaceModule for DriverModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { - // SAFETY: Module init runs once, before either adapter is registered below. - unsafe { binding::init() }; try_pin_init!(Self { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, #[cfg(CONFIG_DRM_ASAHI_G15)] - _g15 <- kernel::driver::Registration::new(c"aurora-g15", module), + _g15 <- kernel::driver::Registration::new(c"asahi-g15", module), _driver <- kernel::driver::Registration::new( ::NAME, module, ), - _g16: register_g16(module)?, // All fallible fields are initialized. This final unit initializer cannot fail. #[cfg(CONFIG_DRM_ASAHI_G15)] _g15_ready: g15::activate(), @@ -108,10 +79,9 @@ impl kernel::InPlaceModule for DriverModule { module! { type: DriverModule, - name: "aurora", + name: "asahi", description: "AGX GPU driver for Apple silicon SoCs", license: "Dual MIT/GPL", - alias: ["asahi"], params: { debug_flags: u64 { default: 0, @@ -173,6 +143,22 @@ module! { default: 3072, description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", }, + g15_pb_blocks: u32 { + default: 64, + description: "G15: PB blocks posted at (re)initialization, 128 KiB each (64..4095)", + }, + g15_pb_reclaim: u32 { + default: 2, + description: "G15: PB reclaim when idle: 0 never, 1 re-initialize, 2 also free backing", + }, + g15_pb_idle_ms: u32 { + default: 1000, + description: "G15: idle time before a PB reclaim, ms (at least 100)", + }, + g15_pb_reclaim_blocks: u32 { + default: 1024, + description: "G15: PB blocks posted above which an idle engine reclaims the PB", + }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", @@ -186,41 +172,5 @@ module! { // permissions: 0, description: "Export AGX firmware stats to sysfs (1 = on, 0 = off)", }, - mem_limit_pct: u32 { - default: 67, - description: "Device-wide GPU memory limit in percent of RAM (0 or >= 100: none)", - }, - client_mem_limit_pct: u32 { - default: 50, - description: "Per-client GPU memory limit in percent of RAM (0: the device limit)", - }, - oom_attribution: u8 { - default: 1, - description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", - }, - g16_deadline_ms: u32 { - default: 5000, - description: "G16: administrative completion-stamp deadline, ms (0 disables; long running work can exceed it)", - }, - g16_compute_deadline_ms: u32 { - default: 5000, - description: "G16: compute completion-stamp deadline, ms (0 permits unbounded compute; firmware recovery stays active)", - }, - g16_repoll_us: u32 { - default: 15, - description: "G16: repoll interval while publication waits for record consumption, us", - }, - g16_resnap_us: u32 { - default: 100, - description: "G16: completion repoll interval, us", - }, - g16_recovery: u32 { - default: 1, - description: "G16: answer qualified firmware recovery handshakes", - }, - g16_stream_gate: u32 { - default: 1, - description: "G16: validate CDM/VDM stream records", - }, }, } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 6227bb4bf815b..43fa17412815d 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,13 +45,11 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { - binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) g16: Option>, + /// The M3 T8122 runtime, stopped on unbind like the G17 GPU. m3: Option, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. @@ -85,18 +83,12 @@ pub(crate) type AsahiDevRef = ARef; pub(crate) type AsahiRegData<'drm> = AsahiDrmRegistrationData<'drm>; /// DRM Driver metadata -/// -/// The major version names the UAPI generation in include/uapi/drm/asahi_drm.h -/// and only changes if that ABI ever breaks. The minor version goes up with -/// each new ioctl, flag or feature bit, so userspace can tell what a kernel -/// accepts without probing. The patch level marks fixes userspace may need to -/// detect. const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { - major: 1, + major: 0, minor: 0, patchlevel: 0, - name: c"aurora", - desc: c"Aurora GPU for Apple AGX", + name: c"asahi", + desc: c"Apple AGX Graphics", }; /// DRM Driver implementation for `AsahiDriver`. @@ -111,6 +103,7 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; @@ -150,82 +143,68 @@ pub(crate) enum ProbeConfig { M3(crate::m3::Chip), } -// OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on -// its entries, so the entries that the native G15 backend owns when it is -// built are passed in by configuration. -macro_rules! asahi_of_device_table { - ($($g15_owned:tt)*) => { - kernel::of_device_table!( - OF_TABLE, - ::IdInfo, - [ - ( - of::DeviceId::new(c"apple,agx-t8140"), - ProbeConfig::G17(&hw::t8140::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6040"), - ProbeConfig::G17(&hw::t6040::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8103"), - ProbeConfig::Legacy(&hw::t8103::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8112"), - ProbeConfig::Legacy(&hw::t8112::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6000"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) - ), - ( - of::DeviceId::new(c"apple,agx-t6001"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) - ), - ( - of::DeviceId::new(c"apple,agx-t6002"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) - ), - ( - of::DeviceId::new(c"apple,agx-t6020"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) - ), - ( - of::DeviceId::new(c"apple,agx-t6021"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) - ), - ( - of::DeviceId::new(c"apple,agx-t6022"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) - ), - ( - of::DeviceId::new(c"apple,agx-t8122"), - ProbeConfig::M3(crate::m3::Chip::T8122) - ), - $($g15_owned)* - ( - of::DeviceId::new(c"apple,agx-t6032"), - ProbeConfig::M3(crate::m3::Chip::T6032) - ), - ] - ); - }; -} - -#[cfg(not(CONFIG_DRM_ASAHI_G15))] -asahi_of_device_table!( - ( - of::DeviceId::new(c"apple,agx-t6030"), - ProbeConfig::M3(crate::m3::Chip::T6030) - ), - ( - of::DeviceId::new(c"apple,agx-t6031"), - ProbeConfig::M3(crate::m3::Chip::T6031) - ), +// OF Device ID table. +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c"apple,agx-t8140"), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8103"), + ProbeConfig::Legacy(&hw::t8103::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8112"), + ProbeConfig::Legacy(&hw::t8112::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6000"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) + ), + ( + of::DeviceId::new(c"apple,agx-t6001"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) + ), + ( + of::DeviceId::new(c"apple,agx-t6002"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) + ), + ( + of::DeviceId::new(c"apple,agx-t6020"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) + ), + ( + of::DeviceId::new(c"apple,agx-t6021"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) + ), + ( + of::DeviceId::new(c"apple,agx-t6022"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) + ), + ( + of::DeviceId::new(c"apple,agx-t8122"), + ProbeConfig::M3(crate::m3::Chip::T8122) + ), + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + ( + of::DeviceId::new(c"apple,agx-t6030"), + ProbeConfig::M3(crate::m3::Chip::T6030) + ), + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + ( + of::DeviceId::new(c"apple,agx-t6031"), + ProbeConfig::M3(crate::m3::Chip::T6031) + ), + ( + of::DeviceId::new(c"apple,agx-t6032"), + ProbeConfig::M3(crate::m3::Chip::T6032) + ), + ] ); -#[cfg(CONFIG_DRM_ASAHI_G15)] -asahi_of_device_table!(); /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { @@ -233,16 +212,19 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + fn unbind<'bound>( + _pdev: &'bound platform::Device>, + this: Pin<&Self::Data<'bound>>, + ) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] if let Some(m3) = this.m3.as_ref() { + // As for G17: the runtime stops its coprocessor through device resources. m3.stop(); } - this.close_binding(); } /// Device probe function. @@ -251,7 +233,6 @@ impl platform::Driver for AsahiPlatformDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { debug::update_debug_flags(); - crate::memacct::LEDGER.configure(); dev_info!(pdev.as_ref(), "Probing...\n"); @@ -263,7 +244,6 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; - // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -280,9 +260,6 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -314,20 +291,12 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); - let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; - // The DRM core created this card's debugfs directory when the device was registered; - // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and - // waits for its readers, before the registration data that owns the snapshot is dropped. - if let Some(snapshot) = debug_stats.as_ref() { - crate::stats_raw::register(reg.device(), snapshot); - } - // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); @@ -335,17 +304,14 @@ impl platform::Driver for AsahiPlatformDriver { let raw_stats = crate::sysfs_exports::register( raw_dev, snapshot, - stats.is_some() && module_parameters::stats_export.value() != 0, + stats.is_some() && *module_parameters::stats_export.value() != 0, )?; let driver = AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, raw_dev, raw_stats, stats, @@ -365,9 +331,6 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -377,13 +340,10 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -402,7 +362,6 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?; let gpu = backend.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); @@ -417,9 +376,6 @@ impl AsahiPlatformDriver { }; dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); Ok(AsahiPlatformDriverData { - binding, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, _device: reg.device().into(), _reg: reg, g17: None, @@ -439,9 +395,6 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -457,75 +410,20 @@ impl AsahiPlatformDriver { return Err(error); } }; - let stats = gpu::Gpu::stats_snapshot(&*gpu); + // G17's firmware channel format has no supported stats decoder yet. let raw_dev = pdev.as_ref().as_raw(); - let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); - let raw_stats = match crate::sysfs_exports::register_g17( - raw_dev, - snapshot, - module_parameters::stats_export.value() != 0, - ) { + let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); return Err(error); } }; - if let Err(error) = - crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) - { - dev_warn!( - pdev.as_ref(), - "idle_off_delay_ms not registered: {:?}\n", - error - ); - } Ok(AsahiPlatformDriverData { - binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), m3: None, - #[cfg(CONFIG_DRM_ASAHI_G16)] - g16: None, - raw_dev, - raw_stats, - stats, - }) - } -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -impl AsahiPlatformDriver { - pub(crate) fn probe_g16<'bound>( - pdev: &'bound platform::Device>, - cfg: &'static crate::g16::Config, - ) -> Result> { - // SAFETY: the qualified UAT output width constrains this device's DMA. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; - let resources = regs::Resources::new(pdev)?; - let res = regs::G16Resources::new(pdev)?; - let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; - // SAFETY: Every adapter calls close_binding in unbind before devres; - // probe errors drop this local owner while the bound callback is live. - let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; - let gpu = crate::g16::Gpu::new(pdev, &drm, cfg, res)?; - let data_gpu = gpu.clone() as Arc; - let data = try_pin_init!(AsahiDrmRegistrationData { - gpu: data_gpu, - resources, - }); - // SAFETY: registration is stored in bound platform data and never forgotten. - let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; - let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = crate::sysfs_exports::register(raw_dev, core::ptr::null(), false)?; - Ok(AsahiPlatformDriverData { - binding, - _device: reg.device().into(), - _reg: reg, - g17: None, - m3: None, - g16: Some(gpu), raw_dev, raw_stats, stats: None, @@ -533,16 +431,9 @@ impl AsahiPlatformDriver { } } -impl AsahiPlatformDriverData<'_> { - pub(crate) fn close_binding(&self) { - self.binding.close(); - } -} - impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { - self.binding.close(); - // `self.raw_dev` was set in `probe` from the platform + // SAFETY: `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 24e8db30e580e..f2f055931d29a 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,8 +16,6 @@ use crate::{ AsahiDrmDevice, // }, gem, - gpu, - memacct, mmu, queue, util::{ @@ -45,7 +43,6 @@ use kernel::sync::{ Mutex, // }; use kernel::time::NSEC_PER_SEC; -use kernel::types::ScopeGuard; use kernel::uaccess::{ UserPtr, UserSlice, // @@ -71,7 +68,6 @@ struct Vm { user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, - gpu: Arc, } impl Drop for Vm { @@ -93,7 +89,6 @@ impl Drop for Vm { pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); } self.vm.bo_deferred_cleanup(); - self.gpu.vm_closed(&self.vm); return; } // When the user Vm is dropped, verify that no user mappings exists. @@ -127,23 +122,10 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, - pub(crate) cumulative: Option, - pub(crate) timeline: bool, } impl SyncItem { - fn parse_one( - file: &DrmFile, - mut data: uapi::drm_asahi_sync, - out: bool, - bundled: bool, - ) -> Result { - if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !out || !bundled || data.timeline_value == 0 { - return Err(EINVAL); - } - data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; - } + fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -164,8 +146,6 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, - cumulative: None, - timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -188,10 +168,12 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: None, + chain_fence: if out { + Some(dma_fence::FenceChain::new()?) + } else { + None + }, timeline_value: data.timeline_value, - cumulative: None, - timeline: true, }) } _ => { @@ -206,52 +188,28 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, - allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; - let mut anchor = None; - // Read each record once so validation and use see the same retained bytes. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); + // SAFETY: We only read this once, so there are no TOCTOU issues. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) })?; // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { - if !allow_cumulative || i < in_count || anchor.is_some() { - return Err(EINVAL); - } - anchor = Some(i as usize); - } - vec.push( - Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, - GFP_KERNEL, - )?; - } - if let Some(anchor) = anchor { - let bundle = crate::g17::sync_output::SignalBundle::new( - &vec[in_count as usize..], - anchor - in_count as usize, - )?; - vec[anchor].cumulative = Some(bundle); - } else { - for sync in &mut vec[in_count as usize..] { - if sync.timeline { - sync.chain_fence = Some(dma_fence::FenceChain::new()?); - } - } + vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; } Ok(vec) @@ -260,7 +218,7 @@ impl SyncItem { #[derive(Clone)] pub(crate) enum Object { - TimestampBuffer(Arc), + TimestampBuffer(Arc), } /// State associated with a client. @@ -275,10 +233,6 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, - /// GPU memory this client created and holds. - mem: Arc, - /// Completed engine time, retained by queues after handle destruction. - usage: Arc, /// Largest size this file has asked GET_PARAMS for the global parameters with /// ([`gpu::Gpu::admit_vm`]). params_size: core::sync::atomic::AtomicU64, @@ -301,35 +255,12 @@ impl drm::file::DriverFile for File { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); let user_range = gpu.user_range(); - let mem = memacct::Client::new()?; - let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) - }) - } - - /// The objects this client created, which `memacct` attributes to it, not every object it - /// has a handle to: an imported or shared buffer is counted once, by its creator. - fn memory_stats(&self) -> Option { - Some(drm::file::MemoryStats { - private: self.mem.committed(), - resident: Some(self.mem.resident()), - ..Default::default() + Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) }) } - fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { - let names = [ - kernel::c_str!("vertex"), - kernel::c_str!("fragment"), - kernel::c_str!("compute"), - ]; - for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { - emit(name, ns); - } - } - fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -339,41 +270,25 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new( - id: u64, - user_range: Range, - mem: Arc, - usage: Arc, - ) -> impl PinInit { - // SAFETY: The initializer supplies aligned, pinned storage for Self. - // XArray::new is infallible; each pinned array is initialized in place, - // then every remaining field is written before returning success. + fn new(id: u64, user_range: Range) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - pin_init::raw_try_init( - raw_vms, - xarray::XArray::>::new(xarray::AllocKind::Alloc1), - )?; + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_vms)?; let raw_queues = addr_of_mut!((*slot).queues); - pin_init::raw_try_init( - raw_queues, - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ), - )?; + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ) + .__pinned_init(raw_queues)?; let raw_objects = addr_of_mut!((*slot).objects); - pin_init::raw_try_init( - raw_objects, - xarray::XArray::>::new(xarray::AllocKind::Alloc1), - )?; + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_objects)?; (*slot).id = id; (*slot).user_range = user_range; - addr_of_mut!((*slot).mem).write(mem); - addr_of_mut!((*slot).usage).write(usage); addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); Ok(()) }) @@ -426,7 +341,7 @@ impl File { let limits = gpu.queue_limits().ok_or(EINVAL)?; let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; UserSlice::new( - UserPtr::from_addr(data.pointer as usize), + UserPtr::from_addr(data.pointer as _), core::mem::size_of_val(&words), ) .writer() @@ -445,11 +360,8 @@ impl File { return Err(ENODEV); } - let memory_limits = memacct::LEDGER.limits(); let mut params = uapi::drm_asahi_params_global { features: 0, - memory_limit: memory_limits.device, - client_memory_limit: memory_limits.client, vm_start: file.inner().user_range.start, vm_end: file.inner().user_range.end, @@ -469,13 +381,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // Write the retained result to this user slice once. + // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); + UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) + core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) })?; file.inner() @@ -490,18 +402,15 @@ impl File { if data.pad != 0 || data.size < size as u64 { return Err(EINVAL); } - let user = UserPtr::from_addr(data.pointer as usize); + let user = UserPtr::from_addr(data.pointer as _); let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; if flags != 0 || pad != 0 { return Err(EINVAL); } let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size).writer().write(&[ - vm_id, - 0, - status.reported_error() as u32, - 0, - ])?; + UserSlice::new(user, size) + .writer() + .write(&[vm_id, 0, status.reported_error() as u32, 0])?; Ok(0) } @@ -512,7 +421,7 @@ impl File { return Err(EINVAL); } let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as usize), size) + UserSlice::new(UserPtr::from_addr(data.pointer as _), size) .reader() .read::<[u32; 4]>()?; if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { @@ -583,22 +492,7 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; - // G16 registers a retained backend owner before returning this VM. Any later - // allocation can fail before file::Vm exists to run its normal close path. - let failed_vm = ScopeGuard::new_with_data( - (vm.clone(), user_range.clone(), kernel_range.clone()), - |(vm, user_range, kernel_range)| { - if gpu.tracked_submission() && vm.status().is_some() && !vm.is_closed() { - vm.close_idle_reclaim(); - if let Err(error) = vm.unmap_user_ranges(user_range, kernel_range) { - pr_err!("VM create: closing unpublished VM failed: {:?}\n", error); - } - vm.bo_deferred_cleanup(); - gpu.vm_closed(&vm); - } - }, - ); + let vm = gpu.new_vm(kernel_range.clone())?; let vm_xa = file.inner().vms(); let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -674,12 +568,10 @@ impl File { user_range, kernel_range, _dummy_mapping: dummy_mapping, - gpu: gpu.clone(), }, GFP_KERNEL, )?)?; - drop(failed_vm.dismiss()); data.vm_id = id; Ok(0) @@ -688,7 +580,7 @@ impl File { /// IOCTL: vm_destroy: Destroy a `Vm`. pub(crate) fn vm_destroy( _device: &AsahiDrmDevice, - _reg_data: &driver::AsahiDrmRegistrationData<'_>, + reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_vm_destroy, file: &DrmFile, ) -> Result { @@ -697,6 +589,7 @@ impl File { Err(ENOENT) } else { drop(vm); + reg_data.gpu.vm_closed(); Ok(0) } } @@ -739,48 +632,11 @@ impl File { None }; - let size = gem::checked_object_size(data.size.try_into()?)?; - let commit = memacct::LEDGER - .admit(&file.inner().mem, size as u64) - .inspect_err(|_| { - cls_pr_debug!( - Errors, - "gem_create: {:#x} bytes refused by the GPU memory limits\n", - size - ); - })?; - // Large G17 user objects are populated in full by the first VM_BIND. - // Do not multiply direct reclaim/compaction latency by their folio - // count. Userspace can evict its BO cache and retry after ENOMEM. - let pages_no_reclaim = - gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); - let gem = gem::new_object( - device, - size, - data.flags, - resv_obj, - reclaim_cpu, - pages_no_reclaim, - commit, - )?; + let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; + if reg_data.gpu.prepare_new_objects() { - #[cfg(CONFIG_DRM_ASAHI_G15)] - if reg_data.gpu.as_any().is::() { - gem::prepare_new_object_g15(&gem)?; - } else { - let registered = device.registration_guard().ok_or(ENODEV)?; - let parent: &kernel::platform::Device = registered.as_ref(); - gem::prepare_new_object(&gem, parent.as_ref())?; - } - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - let registered = device.registration_guard().ok_or(ENODEV)?; - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - { - let parent: &kernel::platform::Device = registered.as_ref(); - gem::prepare_new_object(&gem, parent.as_ref())?; - } + gem::prepare_new_object(&gem)?; } - let handle = gem.create_handle(file)?; data.handle = handle; @@ -857,17 +713,11 @@ impl File { } if let Some(limit) = reg_data.gpu.bind_batch_limit() { - if data.num_binds == 0 - || data.num_binds > limit - || data.userptr == 0 - || data.stride % 8 != 0 - || data.stride as usize > kernel::page::PAGE_SIZE - { - cls_pr_debug!(Errors, "vm_bind: Batch outside the GPU's limits\n"); + if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 + || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { return Err(EINVAL); } } - if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -878,7 +728,7 @@ impl File { let size = usize::try_from(data.stride)? .checked_mul(usize::try_from(data.num_binds)?) .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as usize), size).reader(); + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -896,7 +746,7 @@ impl File { let stride = usize::try_from(data.stride)?; let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as usize), size) + UserSlice::new(UserPtr::from_addr(data.userptr as _), size) .reader() .read_all(&mut bytes, GFP_KERNEL)?; let (vm, user_range, kernel_range) = { @@ -1344,33 +1194,11 @@ impl File { return Err(EINVAL); } - // REALTIME stays reserved. HIGH needs CAP_SYS_NICE or DRM master, as amdgpu and - // panthor require for raised queue priority; EACCES lets userspace fall back to MEDIUM. - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { + // TODO: Allow with CAP_SYS_NICE + if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } - // HIGH has only been run on G17; G13 and G14 keep refusing it. - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !reg_data.gpu.as_any().is::() - { - cls_pr_debug!( - Errors, - "queue_create: HIGH priority is not supported on this GPU\n" - ); - return Err(EINVAL); - } - if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - // SAFETY: capable() reads the current ioctl task's credentials. - && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } - && !file.is_current_master() - { - cls_pr_debug!( - Errors, - "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" - ); - return Err(EACCES); - } let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -1391,7 +1219,6 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, - file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; @@ -1433,9 +1260,6 @@ impl File { } let gpu = ®_data.gpu; - if data.cmdbuf_size as usize > gpu.max_cmdbuf_size() { - return Err(E2BIG); - } if let Some(manager) = gpu.manager() { manager.update_globals(); } @@ -1468,7 +1292,7 @@ impl File { let mut vec = KVec::new(); let copy_commands = |bytes: &mut KVec| { UserSlice::new( - UserPtr::from_addr(data.cmdbuf as usize), + UserPtr::from_addr(data.cmdbuf as _), data.cmdbuf_size as usize, ) .reader() @@ -1480,13 +1304,8 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = SyncItem::parse_array( - file, - data.syncs, - data.in_sync_count, - data.out_sync_count, - gpu.as_any().is::(), - )?; + let syncs = + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; mod_dev_dbg!( device, @@ -1550,7 +1369,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = u64::from(gpu.base_clock_hz()); + let frequency_hz = gpu.base_clock_hz() as u64; let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 66e52e6981197..45b65d1ddc026 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -146,11 +146,6 @@ pub(crate) struct Context<'a> { pub(crate) usc_base: u64, } -struct CheckedContext<'a> { - vm: &'a mut mmu::VmReadGuard, - usc_base: u64, -} - /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are /// checked ([`Prepared::validate`]). pub(crate) struct Prepared<'a> { @@ -366,7 +361,6 @@ impl Timestamps<'_> { .ok_or(ENOENT)? .clone(); let file::Object::TimestampBuffer(mapping) = object; - let mapping = mapping.firmware_mapping()?; let offset = timestamp.offset as usize; if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { return Err(EINVAL); @@ -382,13 +376,7 @@ impl Timestamps<'_> { /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a /// page without the access EACCES. -pub(crate) fn cover( - vm: &mut mmu::VmReadGuard, - start: u64, - end: u64, - read: bool, - write: bool, -) -> Result { +pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result { let mask = mmu::UAT_PGMSK as u64; if end <= start { return Err(EINVAL); @@ -418,11 +406,7 @@ pub(crate) fn cover( } /// Attachment spans, 128-byte granular, actually mapped with `write` or read access. -fn check_attachments( - vm: &mut mmu::VmReadGuard, - list: &[uapi::drm_asahi_attachment], - write: bool, -) -> Result { +fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result { for attachment in list { let end = attachment .pointer @@ -482,8 +466,8 @@ fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) { } /// Per-command render admission. -fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { - let vm = &mut *context.vm; +fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { + let vm = context.vm; let allowed = PROCESS_EMPTY_TILES | NO_VERTEX_CLUSTERING | RSRC_SPEC_HI @@ -567,11 +551,11 @@ fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_rend /// Per-command compute admission. fn check_compute( - context: &mut CheckedContext<'_>, + context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_compute, attachments: &[uapi::drm_asahi_attachment], ) -> Result { - let vm = &mut *context.vm; + let vm = context.vm; let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end); if cmd.flags != 0 || cmd.sampler_count != 0 @@ -665,12 +649,6 @@ impl Prepared<'_> { counts, timestamps, } = self; - // On error, unlock before the last timestamp mapping can drop and unmap. - let mut access = context.vm.read_lock()?; - let mut context = CheckedContext { - vm: &mut access, - usc_base: context.usc_base, - }; let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?; let mut compute = KVec::new(); let mut index = 0; @@ -679,12 +657,12 @@ impl Prepared<'_> { match entry.kind { SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => { // Validated readable, never serialized on G15. - check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?; + check_attachments(context.vm, &attachment_list(entry.payload)?, false)?; } SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?, CMD_RENDER => { let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; - check_render(&mut context, &cmd)?; + check_render(context, &cmd)?; commands.push( Command::Render(KBox::new( RenderCommand { @@ -699,7 +677,7 @@ impl Prepared<'_> { } CMD_COMPUTE => { let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; - check_compute(&mut context, &cmd, &compute)?; + check_compute(context, &cmd, &compute)?; let slots = resolved.get(index).ok_or(EINVAL)?; commands.push( Command::Compute(KBox::new( diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index ae0512de0b156..95f04114ba874 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for -//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting -//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait +//! for the native display and PMP supplier before booting firmware. T6031 retains explicit +//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -45,7 +45,6 @@ mod adt; mod board; mod boot; -mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -144,23 +143,22 @@ const OF_TABLE: kernel::device_id::IdArray = kernel::device_id::IdArray::new([ ( of::DeviceId::new(c_str!("apple,agx-t6030")), - &&hw::t6030::CONFIG, + &hw::t6030::CONFIG, ), ( of::DeviceId::new(c_str!("apple,agx-t6031")), - &&hw::t6031::CONFIG, + &hw::t6031::CONFIG, ), ]); -// Probe admission requires the native display before firmware starts (T6030 -// also its PMP supplier), so both may be loaded at device creation. +// T6030 probe admission requires the native display and PMP supplier before +// firmware starts. T6031 retains explicit loading until it has an equivalent +// display-order admission gate. kernel::of_device_table!( AUTOLOAD_OF_TABLE, - @none, - [ - of::DeviceId::new(c_str!("apple,agx-t6030")), - of::DeviceId::new(c_str!("apple,agx-t6031")), - ] + MODULE_OF_TABLE_G15, + (), + [(of::DeviceId::new(c_str!("apple,agx-t6030")), ())] ); /// The G15 platform adapter: the shared DRM registration probed through @@ -179,12 +177,6 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); - fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { - if let Some(driver) = &this._driver { - driver.close_binding(); - } - } - /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true @@ -219,6 +211,7 @@ impl platform::Driver for Driver<'_> { }, }) } + } /// Firmware starts only after every fallible module/driver registration has succeeded. @@ -381,7 +374,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and the client aliases of every user VM. + /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -446,7 +439,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G15: no aurora-g15 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { @@ -459,18 +452,17 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain()?; + gpu.retain(); Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once; `boot::start` already took it before touching the - /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) -> Result { - boot::glue::retain_module() + pub(crate) fn retain(&self) { + boot::glue::retain_module(); } } @@ -545,21 +537,7 @@ impl Firmware { vms, }, ); - let result = firmware.init_graphs().and_then(|_| { - if cfg.platform != hw::Platform::AdtWithPmp { - return Ok(()); - } - // A report before firmware startup is ACKed without enabling bandwidth scaling. - // Reassert it after setup, including the backend's final device control. - // SAFETY: This session retains its acknowledged T6030 AGX power vote. - let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; - if ret == 0 { - Ok(()) - } else { - Err(Error::from_errno(ret)) - } - }); - if let Err(e) = result { + if let Err(e) = firmware.init_graphs() { let Firmware { session, init, @@ -654,7 +632,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_closed(&self, _vm: &mmu::Vm) { + fn vm_closed(&self) { vm::closed(self) } @@ -669,10 +647,9 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, + queue::Queue::new(self, vm, priority, usc_exec_base)?, GFP_KERNEL, )?) } @@ -681,7 +658,7 @@ impl gpu::Gpu for Gpu { &self, bo: gem::ObjectRef, range: Range, - ) -> Result { - crate::timestamp::Buffer::firmware(timestamp::map(self, bo, range)?) + ) -> Result { + timestamp::map(self, bo, range) } } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 0f91f7c9f0610..7ab3a1919eaec 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,10 +25,8 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) -//! and return them after successful retirement. Failed published jobs retain their slots until -//! their last reference drops (until reboot for jobs retained after a fail-stop). -//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` +//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; @@ -38,12 +36,14 @@ use kernel::{ bindings, device, io::Io, prelude::*, - sync::Arc, time::{delay::fsleep, Delta, Instant, Monotonic}, }; +#[cfg(CONFIG_DRM_ASAHI_G15)] +use kernel::sync::Arc; use super::hw; use super::regs::{mb, rmb, wmb, Mapping}; +#[cfg(CONFIG_DRM_ASAHI_G15)] use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{mmu, pgtable}; @@ -69,6 +69,7 @@ const fn indices(va: u64) -> [usize; 3] { } /// Creates the adopted UAT over the ADT regions (step 2 above). +#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn adopt( dev: &AsahiDevice, cfg: &'static super::Config, @@ -98,7 +99,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(None, pa, count, f) } + unsafe { pgtable::with_root_entries(pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of @@ -201,6 +202,7 @@ pub(crate) struct Window { impl Window { /// Maps the UAT handoff regions ([`Mapping::region`]). + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn new(adt: &super::adt::GpuAdt) -> Result { Self::from_regions(adt.handoff.start, adt.ttbs.start, adt.tables.start, adt.shared_l2.start) } diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index fd6460db2d1fb..d9b0c887deea0 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -78,7 +78,7 @@ impl BufferIds { self.entries .iter() .position(|entry| entry.free()) - .ok_or_else(|| super::capacity::refused(super::capacity::Site::ParameterBuffers))? as u8 + .ok_or(ENOSPC)? as u8 }; if !self.entries[usize::from(id)].free() { return Err(EIO); @@ -503,7 +503,7 @@ impl MetricsLease { break; } } - let slot = selected.ok_or_else(|| super::capacity::refused(super::capacity::Site::MetricsIds))?; + let slot = selected.ok_or(ENOSPC)?; // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { @@ -556,8 +556,6 @@ pub(crate) struct Manager { buffer_id: u8, generation: u64, shrink: Shrink, - /// A tiling pass initialized this buffer manager (25G83 InitBM published). - initialized: bool, } /// Native tiler-heap shrink policy, matching the heap behaviour seen @@ -659,40 +657,14 @@ impl Manager { buffer_id: args.buffer_id, generation: args.generation, shrink: Shrink::default(), - initialized: false, }) } - - /// Whether a published tiling pass initialized this manager (25G83 InitBM). - pub(crate) fn initialized(&self) -> bool { - self.initialized - } - - pub(crate) fn mark_initialized(&mut self) { - self.initialized = true; - } pub(crate) fn buffer_id(&self) -> u8 { self.buffer_id } pub(crate) fn blocks(&self) -> usize { self.tvb.blocks() } - /// Read-only diagnostic while firmware is halted. In particular this - /// does not reset the high-water counter as automatic_target() does. - pub(crate) fn report_recovery(&self) { - let read = |offset| -> Result { - Ok(self.graph.word(offset)?.load(Ordering::Relaxed)) - }; - pr_warn!("G17: recovery tvb id={} blocks={} cursors={:?} pages={:?} capacity={:?} producer={:?} consumer={:?} max_pages={:?} reset={:?} initialized={}\n", - self.buffer_id, self.blocks(), self.cursors(), - read(core::mem::offset_of!(fw::State, pages)), - read(core::mem::offset_of!(fw::State, block_capacity)), - read(core::mem::offset_of!(fw::State, producer)), - read(core::mem::offset_of!(fw::State, consumer)), - read(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages)), - read(fw::CONTROL + core::mem::offset_of!(fw::Control, reset)), - self.initialized); - } pub(crate) fn state_va(&self) -> u64 { self.graph.gpu_va() } @@ -1147,51 +1119,3 @@ fn append_reserved(values: &mut KVec, value: T) { // SAFETY: All callers preflighted this spare entry and initialized it once. unsafe { values.set_len(length + 1) }; } - -impl BufferIds { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "parameter_buffers used={} total={} owned={} referenced={}", - self.entries.iter().filter(|entry| !entry.free()).count(), BUFFER_IDS, - self.entries.iter().filter(|entry| entry.owner).count(), - self.entries.iter().filter(|entry| entry.references != 0).count()) - } -} -impl MetricsIds { - pub(crate) fn used(&self) -> u32 { - self.used.iter().map(|bits| bits.load(Ordering::Acquire).count_ones()).sum::() - - (128 - fw::METRICS_SLOTS as u32) - } -} - -#[kunit_tests(asahi_g17_buffer_capacity)] -mod capacity_tests { - use super::*; - - #[test] - fn failed_build_and_fault_retirement_return_parameter_buffers() { - let mut ids = KBox::init(BufferIds::new(), GFP_KERNEL).unwrap(); - for _ in 0..8 { - let mut clients = KVec::with_capacity(BUFFER_IDS, GFP_KERNEL).unwrap(); - for _ in 0..BUFFER_IDS { - clients.push(ids.allocate_render().unwrap().0, GFP_KERNEL).unwrap(); - } - assert_eq!(ids.allocate_render(), Err(ENOSPC)); - for (index, id) in clients.into_iter().enumerate() { - if index % 2 == 0 { - // Failed constructor never installed the graph owner. - ids.release(id).unwrap(); - } else { - ids.reserve_owner(id).unwrap(); - ids.release(id).unwrap(); - ids.acquire_owner(id).unwrap(); - // Close cannot return a still-owned failed command. - ids.unreserve_owner(id).unwrap(); - assert!(!ids.entries[id as usize].free()); - ids.release(id).unwrap(); - } - } - assert!(ids.entries.iter().all(|entry| entry.free())); - assert_eq!(ids.returned_count, BUFFER_IDS); - } - } -} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index bd96ecc9d460e..ed0f3c71a4c93 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -6,7 +6,7 @@ use super::compact_alias; use crate::g17::{ - fw::render::{RegisterPrograms, RenderPass}, + fw::render::RenderPass, object::{CpuMap, KernelObject}, }; use crate::{ @@ -25,17 +25,6 @@ fn aligned(value: usize, alignment: usize) -> Result { .ok_or(EOVERFLOW) } -/// The 25G83 tiler parameter block: the TPC and seven sub-buffers the tiling registers name, at -/// the default parameter-buffer sizes. Sizes are -/// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. -const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; -const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, GTP6_SIZE, 0x80]; -/// Room for the tiler's records across a partial render. macOS uses 0x1c00, so this is a superset, -/// and the size every partial-render test ran with. -const GTP_SLOTS: usize = 64; -const GTP6_SIZE: usize = GTP_SLOTS - * (((cfg::TVB_MAX_BLOCKS * cfg::TVB_BLOCK_SIZE).div_ceil(0x8000) + 0x17ff) / 24 * 8 + 8); - /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { @@ -44,18 +33,9 @@ pub(crate) struct Layout { pub(crate) used: usize, pub(crate) tpc_size: usize, pub(crate) tvb_blocks: usize, - /// Offsets of the tiler parameter sub-buffers in the TPC object (25G83), `None` for an - /// empty one; the TPC is entry 0. - gtp: [Option; 8], - /// Bytes of the TPC object the command uses. - tpc_end: usize, } impl Layout { - pub(crate) fn new( - pass: &RenderPass, - clusters: u32, - programs: RegisterPrograms, - ) -> Result { + pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { let width = pass.width as usize; let height = pass.height as usize; let layers = pass.layers as usize; @@ -110,51 +90,16 @@ impl Layout { ); } tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); - let mut gtp = [None; 8]; - let mut tpc_end = tpc_size; - if programs == RegisterPrograms::V25G83 { - let sizes = [ - tpc_size, - tilemap_size.checked_mul(64).ok_or(EOVERFLOW)?, - if layers > 1 { 0x4000 } else { 0 }, - 0, - GTP_FIXED_SIZES[0], - GTP_FIXED_SIZES[1], - GTP_FIXED_SIZES[2], - GTP_FIXED_SIZES[3], - ]; - let mut position = 0; - for ((offset, size), alignment) in gtp.iter_mut().zip(sizes).zip(GTP_ALIGNMENTS) { - position = aligned(position, alignment)?; - if size != 0 { - *offset = Some(position); - } - position = position.checked_add(size).ok_or(EOVERFLOW)?; - } - tpc_end = position; - } Ok(Self { layer_offset, heap_offset, used, tpc_size, tvb_blocks, - gtp, - tpc_end, }) } pub(crate) fn tpc_capacity(&self) -> Result { - // TPC_MAX bounds the tail-pointer cache itself. On 25G83 this object - // also contains GTP sub-buffers, notably 64 times the tile map. Applying - // the cache limit to their combined extent rejects valid large targets. - // Keep the original cache and tile-record limits; they also bound the - // derived auxiliary extent before any backing allocation or replacement. - if aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN) > cfg::TPC_MAX - || self.record_stride()? > cfg::TARGET_RECORD_MAX - { - return Err(ERANGE); - } - Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) @@ -203,7 +148,7 @@ impl Target { pub(crate) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; - if stride > cfg::TARGET_RECORD_MAX { + if tpc > cfg::TPC_MAX || stride > cfg::TARGET_RECORD_MAX { return Err(ERANGE); } let records = Object::new(dev, vm, stride * DEPTH)?; @@ -224,21 +169,6 @@ impl Target { self.tpc.address } - /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty - /// or absent one. The TPC object holds them after the TPC. - pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { - if layout.tpc_end > self.tpc.backing.size() { - return Err(ERANGE); - } - let mut addresses = [0; 8]; - for (address, offset) in addresses.iter_mut().zip(layout.gtp) { - if let Some(offset) = offset { - *address = self.tpc.address.checked_add(offset as u64).ok_or(EOVERFLOW)?; - } - } - Ok(addresses) - } - /// Only the span consumed by this render is cleared. A small render does /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { @@ -287,6 +217,9 @@ impl Target { let stride = layout.record_stride()?; let mut grew = false; if tpc > self.tpc.backing.size() { + if tpc > cfg::TPC_MAX { + return Err(ERANGE); + } Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; grew = true; } @@ -362,59 +295,3 @@ impl Target { } } } - - -#[kernel::macros::kunit_tests(asahi_g17_target_capacity)] -mod tests { - use super::*; - - fn pass(size: u32, utile: u8) -> RenderPass { - // SAFETY: RenderPass and its nested surfaces/programs contain only - // integers and bools, for which the all-zero representation is valid. - let mut pass: RenderPass = unsafe { core::mem::zeroed() }; - pass.width = size; - pass.height = size; - pass.layers = 1; - pass.utile_width = utile; - pass.utile_height = utile; - pass - } - - #[test] - fn t6040_auxiliary_tiler_storage_is_not_tail_pointer_cache() { - // CTS point_size_10000 renders to 10240 squared. Its GTP sub-buffer - // is required by the existing 25G83 program, not a larger cache. - for utile in [16, 32] { - let layout = Layout::new(&pass(10240, utile), 2, RegisterPrograms::V25G83).unwrap(); - assert!(layout.tpc_size < cfg::TPC_MAX); - assert!(layout.record_stride().unwrap() < cfg::TARGET_RECORD_MAX); - assert!(layout.tpc_end > cfg::TPC_MAX); - assert!(layout.tpc_capacity().unwrap() >= layout.tpc_end); - assert_eq!(layout.gtp[0], Some(0)); - assert!(layout.gtp[1].unwrap() >= layout.tpc_size); - } - } - - #[test] - fn core_and_tile_record_limits_still_reject_oversized_targets() { - let mut layout = Layout::new(&pass(10240, 32), 2, RegisterPrograms::V25G83).unwrap(); - layout.tpc_size = cfg::TPC_MAX + 1; - assert_eq!(layout.tpc_capacity(), Err(ERANGE)); - let mut layered = pass(16384, 16); - layered.layers = 2; - let layout = Layout::new(&layered, 2, RegisterPrograms::V25G83).unwrap(); - assert!(layout.record_stride().unwrap() > cfg::TARGET_RECORD_MAX); - assert_eq!(layout.tpc_capacity(), Err(ERANGE)); - } - - #[test] - fn neo_keeps_the_same_cache_extent_and_limit() { - let mut layout = Layout::new(&pass(10240, 32), 1, RegisterPrograms::G17P).unwrap(); - assert!(layout.gtp.iter().all(Option::is_none)); - assert_eq!(layout.tpc_capacity().unwrap(), - aligned(layout.tpc_size, mmu::UAT_PGSZ).unwrap().max(cfg::TPC_MIN)); - layout.tpc_size = cfg::TPC_MAX + 1; - layout.tpc_end = layout.tpc_size; - assert_eq!(layout.tpc_capacity(), Err(ERANGE)); - } -} diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs deleted file mode 100644 index 57406e5487a37..0000000000000 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ /dev/null @@ -1,93 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Name capacity refusals at their source, before a common submit or fence -//! failure handler loses their provenance. Counters are device-lifetime -//! independent diagnostics, never admission or ownership state. - -use core::sync::atomic::{AtomicU64, Ordering}; -use kernel::prelude::*; - -#[derive(Clone, Copy, Debug)] -pub(super) enum Site { - ComputeReserve, - PhysicalQids, - RenderSlots, - LogicalQueues, - ComputeSlots, - QosIds, - ComputeItems, - RetiredRenderBindings, - RenderPoolIds, - RenderWorkSlots, - ParameterBuffers, - MetricsIds, - LowerVa, - UpperVa, - AliasVa, - ExecutionContexts, -} - -const SITE_COUNT: usize = Site::ExecutionContexts as usize + 1; -static REFUSALS: [AtomicU64; SITE_COUNT] = [const { AtomicU64::new(0) }; SITE_COUNT]; - -/// First and power-of-two refusals per site keep a pressure storm bounded in -/// the log. This does not change whether the caller retries or rejects work. -#[track_caller] -pub(super) fn refused(site: Site) -> Error { - let count = REFUSALS[site as usize].fetch_add(1, Ordering::Relaxed).wrapping_add(1); - if count.is_power_of_two() { - let at = core::panic::Location::caller(); - pr_warn!("G17 capacity refusal {:?} count={} at {}:{}\n", site, count, - at.file().rsplit("asahi/").next().unwrap_or(at.file()), at.line()); - } - ENOSPC -} - -/// Read-only diagnostics hold the live device session while sampling its pools. -/// They never wake firmware, reclaim a resource, or change admission state. -pub(super) struct View(pub(super) kernel::sync::Arc); - -impl kernel::debugfs::Writer for View { - fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "version=1 sampled_ns={} crashed={}", super::now_ns(), - u8::from(self.0.crashed.load(Ordering::Acquire)))?; - let uat = { - let state = self.0.state.lock(); - let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; - writeln!(f, "online=1")?; - let (converted, published) = super::sync_input::counters(); - writeln!(f, "fw_inputs converted={} published={} native={} pending={}", - converted, published, super::sync_input::native_count(), - super::sync_input::pending_count())?; - writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?; - firmware.queues.write_capacity(f)?; - firmware.pool.write_capacity(f)?; - writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; - firmware.uat.clone() - }; - // Mapping destruction can issue firmware flushes while holding VM - // locks. Never take a VM/TTBAT lock with the device mutex held here. - // The retained UAT owns valid allocator metadata even during shutdown. - writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; - let mut upper = uat.geometry().kernel_range(); - upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; - for (name, vm, range) in [ - ("dynamic_lower_bytes", uat.kernel_lower_vm(), crate::hw::t8140::dynamic::LOWER), - ("dynamic_upper_bytes", uat.kernel_vm(), upper), - ] { - match vm.capacity(range.clone()) { - Ok((used, largest)) => writeln!(f, "{} used={} total={} largest_hole={}", name, used, range.end - range.start, largest)?, - Err(error) => writeln!(f, "{} unavailable={:?}", name, error)?, - } - } - for (site, count) in [ - Site::ComputeReserve, Site::PhysicalQids, Site::RenderSlots, Site::LogicalQueues, - Site::ComputeSlots, Site::QosIds, Site::ComputeItems, Site::RetiredRenderBindings, - Site::RenderPoolIds, Site::RenderWorkSlots, Site::ParameterBuffers, Site::MetricsIds, - Site::LowerVa, Site::UpperVa, Site::AliasVa, Site::ExecutionContexts, - ].iter().zip(REFUSALS.iter()) { - writeln!(f, "refusals site={:?} count={}", site, count.load(Ordering::Relaxed))?; - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index c65f7de6ac915..3dfa760969224 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -12,8 +12,6 @@ use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; use kernel::{prelude::*, uapi}; -const _: () = assert!(uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_AS_BUILD == 1 << 2); - const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; @@ -123,7 +121,7 @@ impl<'a> Parser<'a> { let bytes = self.bytes.get(start..end).ok_or(EINVAL)?; self.bytes = &self.bytes[end..]; let barriers = [header.vdm_barrier, header.cdm_barrier]; - match u32::from(header.cmd_type) { + match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER | uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { if barriers.iter().zip(self.counts).any(|(&barrier, count)| { @@ -133,7 +131,7 @@ impl<'a> Parser<'a> { { return Err(EINVAL); } - let payload = if u32::from(header.cmd_type) + let payload = if header.cmd_type as u32 == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { self.counts[0] += 1; @@ -240,7 +238,7 @@ impl UscWindow { } require_range( space, - self.base + u64::from(tagged_offset & !7), + self.base + (tagged_offset & !7) as u64, 4, Access::Read, ) @@ -308,14 +306,12 @@ fn surface( #[allow(clippy::large_enum_variant)] pub(crate) enum Validated { Render { - long_job: bool, pass: RenderPass, ray_query: [RayQueryStage; 2], attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { - long_job: bool, intersection: IntersectionTables, scratch: ScratchRequest, usc_base: u64, @@ -327,19 +323,6 @@ pub(crate) enum Validated { }, } -impl Validated { - pub(crate) fn long_job(&self) -> bool { - match self { - Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job, - } - } - pub(crate) fn allow_long_job(&mut self) { - match self { - Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job = true, - } - } -} - impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { @@ -351,7 +334,6 @@ impl Payload { pub(crate) fn validate( &self, window: &UscWindow, - variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { match self { @@ -425,7 +407,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, - few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, @@ -458,11 +440,6 @@ impl Payload { rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; for stage in ray_query { - if (stage.intersection.enabled() || stage.scratch.enabled()) - && !variant.ray_traversal - { - return Err(refuse_ray_traversal()); - } if stage.intersection.enabled() { require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; @@ -470,8 +447,6 @@ impl Payload { } } Ok(Validated::Render { - long_job: super::timeout::select(false, - ray_query.map(|stage| stage.intersection.enabled()), false), pass, ray_query: *ray_query, attachments: *attachments, @@ -479,8 +454,8 @@ impl Payload { }) } Self::Compute(cmd, attachments, intersection) => { - let as_build = super::timeout::as_build(cmd.flags).ok_or(EINVAL)?; - if cmd.cdm_ctrl_stream_base & 3 != 0 + if cmd.flags != 0 + || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base || !helper_is_empty(&cmd.helper) @@ -495,19 +470,14 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; - let scratch = ScratchRequest::new(cmd.scs_layout)?; - if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { - return Err(refuse_ray_traversal()); - } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; if code != 0 { require_range(space, code, 64, Access::Read)?; } } Ok(Validated::Compute { - long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), intersection: *intersection, - scratch, + scratch: ScratchRequest::new(cmd.scs_layout)?, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, @@ -519,9 +489,3 @@ impl Payload { } } } - -/// Ray-traversal work on a firmware build that does not run it ([`super::fw::Variant`]). -fn refuse_ray_traversal() -> Error { - kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); - Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) -} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 90f6d77780bff..a3888f98231db 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -13,12 +13,7 @@ use super::{ recovery::Class, status::VmStatus, }; -use crate::{ - driver::AsahiDrmDevice, - file, - hw::t8140::{dynamic, resources as cfg}, - mmu, -}; +use crate::{driver::AsahiDrmDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, @@ -32,9 +27,9 @@ mod publication; pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { - // The buffer keeps its GPU alias and its CPU mapping alive. - buffer: Arc, - offset: usize, + // Keep the existing GPU alias alive along with its prepared CPU view. + _mapping: Arc, + cpu: mmu::TimestampMapping, } impl Destination { @@ -50,13 +45,13 @@ impl Destination { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(buffer) = object; + let file::Object::TimestampBuffer(mapping) = object; let offset: usize = timestamp.offset.try_into()?; - if offset.checked_add(size_of::()).ok_or(EINVAL)? > buffer.size() { + if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { return Err(ERANGE); } - let offset = buffer.timestamp_offset(offset)?; - Ok(Some(Self { buffer, offset })) + let cpu = mapping.timestamp_mapping(offset)?; + Ok(Some(Self { _mapping: mapping, cpu })) } fn write(&self, ticks: u64) { @@ -66,14 +61,13 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.buffer.write_timestamp(self.offset, ns); + self.cpu.write(ns); barrier(Ordering::Release); } } -/// Timestamp destinations resolved and retained before any command is enqueued. Dropping -/// one releases only a buffer reference; the buffer defers its own mapping release. -pub(crate) struct Destinations([Option; 4]); +/// Timestamp mappings resolved and retained before any command is enqueued. +pub(crate) struct Destinations(Option; 4]>>); impl Destinations { pub(crate) fn resolve( @@ -88,15 +82,20 @@ impl Destinations { destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; } - Ok(Self(destinations)) - } - - pub(crate) fn has_outputs(&self) -> bool { - self.0.iter().any(Option::is_some) + // Empty submissions need no cleanup allocation. Timestamped commands + // prepare the unique work item before any output fence is published. + let retained = if destinations.iter().any(Option::is_some) { + Some(crate::cleanup::Deferred::new(destinations)?) + } else { + None + }; + Ok(Self(retained)) } fn iter(&self) -> core::slice::Iter<'_, Option> { - self.0.iter() + let destinations: &[Option] = + self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); + destinations.iter() } } @@ -145,8 +144,7 @@ impl HostOutput { } } -/// Firmware writes two aligned words in a write-combined page of its own per -/// compute command. +/// Firmware writes two aligned words in a fresh write-combined page per compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. mapping: mmu::KernelMapping, @@ -168,84 +166,6 @@ impl ComputeTimestamps { } } -/// Compute timestamp pages of one context, kept mapped for its next commands. -/// -/// A fresh page costs a GEM object, its write-combined kernel mapping and a -/// GPU mapping on the submission path, and their teardown when the command's -/// completion is dropped. A page comes back here only from that drop, which -/// follows exact retirement or a kill witness, so the firmware no longer -/// writes it; both words are cleared before the page is handed out again. -#[pin_data] -pub(crate) struct ComputeTimestampCache { - #[pin] - pages: Mutex>, -} - -impl ComputeTimestampCache { - /// Pages retained per context; deeper pipelines allocate the rest. - const RETAINED: usize = 8; - - pub(crate) fn new() -> Result> { - let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?; - Arc::pin_init( - pin_init!(Self { - pages <- new_mutex!(pages, "G17 compute timestamp pages"), - }), - GFP_KERNEL, - ) - } - - fn take(self: &Arc, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { - let reused = self.pages.lock().pop(); - let page = match reused { - Some(page) => { - page.object.dword(0)?.store(0, Ordering::Relaxed); - page.object.dword(8)?.store(0, Ordering::Relaxed); - barrier(Ordering::SeqCst); - page - } - None => ComputeTimestamps::new(dev, vm)?, - }; - Ok(PooledTimestamps { - page: Some(page), - cache: self.clone(), - }) - } - - fn put(&self, page: ComputeTimestamps) { - let rejected = { - let mut pages = self.pages.lock(); - pages.push_within_capacity(page).err() - }; - // Teardown of a page beyond the retained count runs unlocked. - drop(rejected); - } -} - -/// A compute timestamp page lent to one command, returned to its context's -/// cache when the command's completion is dropped. -struct PooledTimestamps { - page: Option, - cache: Arc, -} - -impl PooledTimestamps { - fn page(&self) -> Result<&ComputeTimestamps> { - self.page.as_ref().ok_or(EINVAL) - } -} - -impl Drop for PooledTimestamps { - fn drop(&mut self) { - if let Some(page) = self.page.take() { - self.cache.put(page); - } - } -} - -/// Republications of one command whose earlier publications firmware dropped unexecuted. -pub(crate) const MAX_REPLAYS: u8 = 3; - #[pin_data] pub(crate) struct Completion { status: Arc, @@ -253,10 +173,8 @@ pub(crate) struct Completion { // Present only when this command's public result may wait for a render // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, - compute: Option, + compute: Option, feed: Option>, - usage: Arc, - sources: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -267,21 +185,11 @@ pub(crate) struct Completion { timeout_error: AtomicI32, replays: AtomicU8, replay_timeout: AtomicBool, - /// A recovery blamed this command. Unlike `class`, which a later - /// classification can change, this never clears. - blamed: AtomicBool, - /// A firmware restart listed this command as in flight. Until a replay - /// republishes it from the start, it never reports success. - interrupted: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } impl Completion { - #[expect( - clippy::too_many_arguments, - reason = "Completion construction takes distinct queue, context, and lifetime owners." - )] pub(crate) fn new( dev: &AsahiDrmDevice, contexts: &FenceContexts, @@ -292,28 +200,23 @@ impl Completion { compute: bool, host_predecessors: bool, feed: Option>, - sources: Option>, - usage: Arc, ) -> Result> { let fence = fence::independent(contexts)?; - let ordered = !compute && (host_predecessors || destinations.iter().any(Option::is_some)); + let ordered = !compute + && (host_predecessors || destinations.iter().any(Option::is_some)); let terminal = if host_predecessors { Some(fence::independent(contexts)?) } else { None }; - let output = Output { - fence, - member, - destinations, - }; + let output = Output { fence, member, destinations }; let output = if ordered { HostOutput::Ordered(Publication::new(output, context.status().clone())?) } else { HostOutput::Immediate(output) }; let compute = if compute { - Some(context.compute_timestamps().take(dev, context.vm())?) + Some(ComputeTimestamps::new(dev, context.vm())?) } else { None }; @@ -324,8 +227,6 @@ impl Completion { terminal, compute, feed, - usage, - sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -334,8 +235,6 @@ impl Completion { timeout_error: AtomicI32::new(0), replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), - blamed: AtomicBool::new(false), - interrupted: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -347,9 +246,7 @@ impl Completion { Fence::from_fence(&self.output.output().fence) } pub(crate) fn scheduler_fence(&self) -> Fence { - self.terminal - .as_ref() - .map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) + self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) } pub(crate) fn publication(&self) -> Option> { match &self.output { @@ -373,62 +270,22 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } - pub(super) fn sources(&self) -> Option<&Arc> { - self.sources.as_ref() - } - pub(super) fn is_input_source(&self, source: &Arc) -> bool { - self.output.output().member.is_source(source) - } - pub(super) fn set_source_ready(&self, ready: Fence) { - self.output.output().member.set_source_ready(ready); - } - pub(super) fn needs_source_ready(&self) -> bool { - self.output.output().member.needs_source_ready() - } - pub(super) fn note_input_error(&self, error: Error) -> bool { - self.output.output().member.note_error(error) - } - /// Called under the device mutex as a successful retirement is queued. - pub(super) fn note_source_retired(&self) { - if self.interrupted() { - // complete() fails it; consumers must not take its output as valid. - self.note_input_error(ECANCELED); - return; - } - match self.check_source_inputs() { - Ok(()) => self.output.output().member.note_retired(), - Err(error) => { - self.note_input_error(error); - } - } - } - pub(super) fn check_source_inputs(&self) -> Result { - if let Some(sources) = &self.sources { - sources.check()?; - } - Ok(()) - } - pub(crate) fn work_state(&self) -> Result> { self.output.output().member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { - self.output - .output() - .destinations - .iter() - .any(Option::is_some) + self.output.output().destinations.iter().any(Option::is_some) } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) + Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -448,14 +305,11 @@ impl Completion { let accepted = self .replays .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { - (count < MAX_REPLAYS).then_some(count + 1) + (count < 3).then_some(count + 1) }) .is_ok(); if accepted { self.replay_timeout.store(true, Ordering::Release); - // Callers replay only work proven not to have started; the new - // publication runs from the beginning and reports its own result. - self.interrupted.store(false, Ordering::Release); } accepted } @@ -481,7 +335,7 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -530,49 +384,6 @@ impl Completion { self.class.load(Ordering::Acquire) == 1 } - pub(crate) fn guilty(&self) -> bool { - self.class.load(Ordering::Acquire) == 2 - } - - /// A recovery blamed this command, whatever class it keeps: it may have caused the fault, - /// so it is never published again. - pub(crate) fn note_blamed(&self) { - self.blamed.store(true, Ordering::Release); - } - - pub(crate) fn blamed(&self) -> bool { - self.blamed.load(Ordering::Acquire) - } - - /// A firmware restart listed this command in its recovery table: it was in - /// flight, so it may have run partly. Its stamps and completion record may - /// still look like success. Unless a replay republishes it from the start, - /// it fails and the failure is always reported to the client. - pub(crate) fn note_interrupted(&self) { - self.interrupted.store(true, Ordering::Release); - } - - pub(crate) fn interrupted(&self) -> bool { - self.interrupted.load(Ordering::Acquire) - } - - /// Whether this failure makes the VM unusable. Spared work, and interrupted - /// work no recovery found guilty, fail without poisoning it. - pub(crate) fn poisons(&self, error: Error) -> bool { - let interrupted = self.interrupted(); - (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) - } - - /// ECANCELED stays silent only for work that was never interrupted (closed - /// or cancelled by its owner); every other failure reaches the client. - fn report(&self, error: Error) { - if self.poisons(error) { - self.status.record(error); - } else if error != ECANCELED || self.interrupted() { - self.status.report_failure(error); - } - } - /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -590,12 +401,6 @@ impl Completion { Ok(()) } - /// An ioctl publication that met contention returns its lease before the - /// scheduler publishes the command. Nothing referred to it yet. - pub(crate) fn detach_render_lease(&self) -> Option { - self.render.lock().take() - } - pub(crate) fn render_slot(&self) -> Option { self.render .lock() @@ -603,26 +408,16 @@ impl Completion { .map(super::queue::admission::RenderLease::slot) } - /// Called under the device mutex once the render's kicks are committed. - pub(crate) fn note_render_publication(&self, fragment: super::fw::kick::KickDependency) { - if let Some(lease) = self.render.lock().as_ref() { - lease.note_publication(fragment); - } - } - /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { - lease.quarantine(self.spared()); + lease.quarantine(); } } /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { - if let Some(sources) = &self.sources { - sources.release(); - } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -631,15 +426,7 @@ impl Completion { drop(lease); } - /// Whether the command still holds its VM pin: firmware may still own or replay it. - pub(crate) fn retains_vm(&self) -> bool { - self.vm_job.lock().is_some() - } - fn release_healthy(&self) { - if let Some(sources) = &self.sources { - sources.release(); - } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -647,40 +434,19 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { - if result.is_ok() { - if let Err(error) = self.check_source_inputs() { - result = Err(error); - } - } + pub(crate) fn complete(&self, result: Result<[u64; 4]>) { if self.signalled.swap(true, Ordering::AcqRel) { return; } - if result.is_ok() && self.interrupted() { - result = Err(ECANCELED); - } - if let Ok(ticks) = &result { - use crate::usage::Engine; - let hz = cfg::COMMAND_TIMESTAMP_HZ as u32; - if self.compute.is_some() { - self.usage.add(Engine::Compute, ticks[0], ticks[1], hz); - } else { - self.usage.add(Engine::Vertex, ticks[0], ticks[1], hz); - self.usage.add(Engine::Fragment, ticks[2], ticks[3], hz); - } - } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { - // Compute reports its dispatch envelope in the first pair, a - // render its fragment pass in the second. - if self.compute.is_some() { - feed.note_pass(ticks[0], ticks[1]); - } else { - feed.note_pass(ticks[2], ticks[3]); - } + feed.note_render_pass(ticks[2], ticks[3]); } if let Err(error) = result { - // Spared work cancelled after another VM's recovery is not this client's device loss. - self.report(error); + if self.spared() { + self.status.report_failure(error); + } else if error != ECANCELED { + self.status.record(error); + } } self.release_healthy(); self.signal_terminal(result); @@ -694,9 +460,7 @@ impl Completion { return; } self.status.record_if_device_loss(error); - // As in complete(): a cancellation is not this client's device loss. A guilty VM - // already carries its own sticky error. - if self.accepted.load(Ordering::Acquire) && error != ECANCELED { + if self.accepted.load(Ordering::Acquire) { self.status.report_failure(error); } self.release_healthy(); @@ -715,11 +479,9 @@ impl Completion { guard.quarantine(); } if self.spared() { - if error != ECANCELED || self.interrupted() { - self.status.report_failure(error); - } - } else { - self.report(error); + self.status.report_failure(error); + } else if error != ECANCELED { + self.status.record(error); } self.signal_terminal(Err(error)); self.output.finish(Err(error)); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 8a6d9b572dec0..6bc62a7d44bee 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -14,9 +14,8 @@ use super::{ status::VmStatus, }; use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; -use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}; -use identity::{Order, RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; -use super::job::Engine; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; use kernel::{ new_mutex, prelude::*, @@ -40,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(super::capacity::refused(super::capacity::Site::QosIds)); + return Err(ENOSPC); } *used |= 1u128 << id; Ok(QosLease { @@ -82,19 +81,6 @@ impl WorkStorage { fence(Ordering::Release); Ok(()) } - /// Whether the firmware completed every kick counted so far on node `word`. - fn drained(&self, word: u32) -> Result { - let slot = (word & 63) as usize; - if self.words.get(slot) != Some(&Some(word)) { - return Err(EINVAL); - } - let completed = self - .object - .word(slot * NODE_STRIDE + core::mem::offset_of!(fw::WorkNode, completed))? - .load(Ordering::Relaxed); - fence(Ordering::Acquire); - Ok(completed >= self.counts[slot]) - } fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { let slot = (word & 63) as usize; if self.words.get(slot) != Some(&Some(word)) { @@ -128,39 +114,18 @@ pub(crate) struct Context { scheduler: Arc, scheduler_generation: u32, published: AtomicBool, - /// A kill of this context was published, through a compute queue or on its render pairs' - /// behalf. A context is killed at most once. - kill_published: AtomicBool, - /// Advisory only: publication still checks the physical binding under the - /// device mutex. False skips an inline attempt known to need admission. - compute_binding_hint: AtomicBool, - /// Mixed-engine contexts keep the original inline publication path. - render_seen: AtomicBool, qos: QosLease, owner_pid: u32, states: WorkStates, - /// Signalled when a work state returns; publishers that found none wait on it. - state_returns: Arc, - /// The domain refused compute publishers wait on; signalled with `state_returns` when the - /// admission turn moves on. - compute_returns: Arc, - /// The order in which this context's ioctls take work-state nodes. - order: Arc>, work: Arc>, work_base: u64, render: Arc>>>, render_blocks: AtomicUsize, - /// Timestamp pages of retired compute commands, mapped in this context's VM. - compute_timestamps: Arc, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, } impl Context { - #[expect( - clippy::too_many_arguments, - reason = "A context takes its allocators, identities and both admission domains." - )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -168,8 +133,6 @@ impl Context { execution: Arc, owner_pid: u32, policy: Policy, - state_returns: Arc, - compute_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -218,53 +181,25 @@ impl Context { GFP_KERNEL, )?; let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?; - let order = - Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { execution, status, scheduler, scheduler_generation, published: AtomicBool::new(false), - kill_published: AtomicBool::new(false), - compute_binding_hint: AtomicBool::new(false), - render_seen: AtomicBool::new(false), qos, owner_pid, states: WorkStates::new(), - state_returns, - compute_returns, - order, work, work_base, render, render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), - compute_timestamps: super::completion::ComputeTimestampCache::new()?, policy, }) } pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } - pub(crate) fn compute_binding_hint(&self) -> bool { - self.render_seen.load(Ordering::Acquire) - || self.compute_binding_hint.load(Ordering::Relaxed) - } - /// This logical queue has submitted render work, so its compute may have a render prefix. - pub(crate) fn render_seen(&self) -> bool { - self.render_seen.load(Ordering::Acquire) - } - pub(crate) fn note_render(&self) { - if !self.render_seen.load(Ordering::Relaxed) { - self.render_seen.store(true, Ordering::Release); - } - } - pub(crate) fn note_compute_binding(&self, owned: bool) { - self.compute_binding_hint.store(owned, Ordering::Relaxed); - } - pub(crate) fn compute_timestamps(&self) -> &Arc { - &self.compute_timestamps - } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -312,28 +247,6 @@ impl Context { _work: self.work.clone(), } } - /// Called once the firmware consumed this context's scheduler release. If its work list - /// still links a node, the firmware keeps referencing that node: on Neo a render command - /// that ran behind a failed compute command is never unlinked, and a later context given - /// the same work storage (the object cache hands it out again) made the firmware recover - /// the GPU. Keep the storage mapped and out of reuse until firmware stop; returns whether - /// it was linked. - pub(crate) fn retain_linked_work(&self, pool: &Arc) -> bool { - // An unreadable head counts as linked: retaining is the safe side. - let linked = self.work_head().map_or(true, |head| head != [0, self.work_head_va()]); - if linked { - self.work.lock().object.retain_until_stop(pool); - } - linked - } - - /// The firmware may still walk this context's work list into memory it names: a node is - /// linked and the scheduler page shows no kill. Unreadable state counts as linked. - pub(crate) fn firmware_may_link(&self) -> bool { - let killed = super::queue::compute::exit::State::read(self).map_or(0, |state| state.killed); - must_retain(self.work_head().ok(), self.work_head_va(), killed) - } - pub(crate) fn work_head_va(&self) -> u64 { self.work_base + (UAT_PGSZ + 0x400) as u64 } @@ -348,16 +261,6 @@ impl Context { Ok(head) } - /// Record that a kill of this context was published (see `kill_published`). - pub(crate) fn note_kill_published(&self) { - self.kill_published.store(true, Ordering::Release); - } - - /// A kill of this context was published. - pub(crate) fn kill_published(&self) -> bool { - self.kill_published.load(Ordering::Acquire) - } - pub(crate) fn is_published(&self) -> bool { self.published.load(Ordering::Acquire) } @@ -424,60 +327,41 @@ impl Context { pub(crate) struct WorkStateLease { context: Arc, word: AtomicU32, - /// Index of the engine whose admission took the node, valid once `word` is set. - engine: AtomicU8, - /// The ioctl's place in its context's submission order. - ordinal: u64, } impl WorkStateLease { - pub(super) fn status(&self) -> &Arc { self.context.status() } pub(crate) fn deferred(context: Arc) -> Self { - let ordinal = context.order.lock().assign(); Self { context, word: AtomicU32::new(FREE), - engine: AtomicU8::new(0), - ordinal, } } - /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: too many - /// older ioctls still wait for theirs (see `identity::Order`), or no node may be taken now - /// (see `identity::may_take`); an older ioctl's admission or a return lets it try again. - pub(crate) fn acquire(&self, engine: Engine) -> Result { - let current = self.word(); - if current != FREE { - return Ok(current); - } - // The work-storage lock serializes the decision with this context's other - // admissions, and with the other engine of this ioctl taking its node. - let mut work = self.context.work.lock(); + pub(crate) fn acquire(&self) -> Result { let current = self.word(); if current != FREE { return Ok(current); } - let mut order = self.context.order.lock(); - if !order.admits(self.ordinal) { - return Err(EBUSY); - } - let engine = engine.index(); - let candidate = self.context.states.allocate(engine).ok_or(EBUSY)?; - if let Err(error) = work.initialize(candidate) { - self.context.release_state(candidate, engine); + let Some(candidate) = self.context.states.allocate() else { + let current = self.word(); + return if current != FREE { + Ok(current) + } else { + Err(EBUSY) + }; + }; + if let Err(error) = self.context.work.lock().initialize(candidate) { + self.context.states.release(candidate); return Err(error); } - self.engine.store(engine as u8, Ordering::Relaxed); - self.word.store(candidate, Ordering::Release); - let wake = order.admit(self.ordinal); - drop(order); - drop(work); - if wake { - self.context.pass_turn(); + match self + .word + .compare_exchange(FREE, candidate, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => Ok(candidate), + Err(installed) => { + self.context.states.release(candidate); + Ok(installed) + } } - Ok(candidate) - } - /// The domain a publisher samples before acquire() and waits on after EBUSY. - pub(crate) fn returns(&self) -> &Arc { - &self.context.state_returns } pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) @@ -489,16 +373,6 @@ impl WorkStateLease { } Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) } - /// Whether the firmware completed every kick counted so far on this ioctl's node. It stores - /// the completion stamps of the node's commands only then, so a command whose record is - /// present still waits for its stamp while other commands of its ioctl run. - pub(crate) fn drained(&self) -> Result { - let word = self.word(); - if word == FREE { - return Err(EINVAL); - } - self.context.work.lock().drained(word) - } pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { self.context .work @@ -513,8 +387,7 @@ impl WorkStateLease { } pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or_else(|| - super::capacity::refused(super::capacity::Site::RenderWorkSlots)) + self.context.work.lock().render.get(pair).ok_or(ENOSPC) } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context @@ -529,75 +402,7 @@ impl Drop for WorkStateLease { fn drop(&mut self) { let word = self.word(); if word != FREE { - self.context.release_state(word, usize::from(self.engine.load(Ordering::Relaxed))); - } else if self.context.order.lock().leave(self.ordinal) { - self.context.pass_turn(); - } - } -} - -impl Context { - /// Every return bumps the domain: a publisher that sampled it before its - /// failed acquire() then never sleeps through this node. - fn release_state(&self, word: u32, engine: usize) { - if self.states.release(word, engine) { - self.state_returns.notify(); - } - } - /// A refused publisher waits on its engine's domain. Bumping both once the turn moved past a - /// refusal means one that sampled before its refused acquire() never sleeps through it. - fn pass_turn(&self) { - self.state_returns.notify(); - self.compute_returns.notify(); - } -} - -impl QosIds { - pub(crate) fn used(&self) -> u32 { - (*self.used.lock() & !3u128).count_ones() - } -} - -/// Memory a context's work list may lead the firmware to stays allocated: a linked node (or -/// an unreadable head) and no acknowledged kill. -fn must_retain(head: Option<[u64; 2]>, head_va: u64, killed: u32) -> bool { - killed == 0 && head.is_none_or(|head| head != [0, head_va]) -} - -#[kunit_tests(asahi_g17_qos_capacity)] -mod capacity_tests { - use super::*; - - #[test] - fn linked_unkilled_work_is_retained() { - let va = 0xfffffc20_0000_4400; - assert!(!must_retain(Some([0, va]), va, 0)); - assert!(must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 0)); - assert!(must_retain(None, va, 0)); - assert!(!must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 1)); - assert!(!must_retain(None, va, 1)); - } - - #[test] - fn failed_client_construction_returns_every_qos_id() { - let ids = QosIds::new().unwrap(); - for _ in 0..8 { - let mut clients = KVec::with_capacity(126, GFP_KERNEL).unwrap(); - for _ in 0..126 { - clients.push(ids.acquire().unwrap(), GFP_KERNEL).unwrap(); - } - assert_eq!(ids.used(), 126); - assert!(matches!(ids.acquire(), Err(ENOSPC))); - drop(clients); - assert_eq!(ids.used(), 0); - assert_eq!(*ids.used.lock(), 3); + self.context.states.release(word); } } } - -impl Context { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>, scope: &str, index: usize) -> kernel::fmt::Result { - writeln!(f, "context scope={} index={} id={} work_used={} work_total={} render_used={} render_total=2", - scope, index, self.id(), self.states.used(), STATE_COUNT, self.work.lock().render.used()) - } -} diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 7586f8474e602..6c48367e2d686 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -4,50 +4,23 @@ //! the complete sequence distinguishes its owner until exact retirement. use core::sync::atomic::{AtomicU32, Ordering}; -use kernel::prelude::*; pub(super) const FREE: u32 = u32::MAX; pub(super) const STATE_COUNT: usize = 63; pub(super) const NODE_STRIDE: usize = 0x100; -/// Engines whose admissions take work-state nodes: render (0) and compute (1). -pub(super) const ENGINES: usize = 2; - -/// Whether an engine that holds `held` nodes may take one of `free` nodes. -/// -/// All engines of one ioctl share its node, and the node returns only once every command of -/// the ioctl has retired. So render admission can keep taking nodes for newer ioctls while -/// their compute commands still wait in the compute queue behind an older ioctl that has no -/// node yet; once render held every node, nothing could retire and both queues stalled. An -/// engine therefore never takes the last free node while it already holds one. The other -/// engine can then always take a node, and an engine refused here waits only for nodes it -/// took itself, for ioctls ahead of its waiting command. Each engine admits a context's -/// ioctls in submission order, so those waits cannot form a cycle. -pub(super) const fn may_take(free: usize, held: u32) -> bool { - free > 1 || (free == 1 && held == 0) -} - pub(super) struct WorkStates { next: AtomicU32, live: [AtomicU32; 64], - /// Nodes taken by each engine's admissions and not yet released. - held: [AtomicU32; ENGINES], } impl WorkStates { pub(super) const fn new() -> Self { Self { next: AtomicU32::new(0), live: [const { AtomicU32::new(FREE) }; 64], - held: [const { AtomicU32::new(0) }; ENGINES], } } - /// Takes a node for `engine`, or none when [`may_take`] refuses. Callers serialize - /// allocations of one context; releases may run concurrently. - pub(super) fn allocate(&self, engine: usize) -> Option { - let held = self.held.get(engine)?; - if !may_take(STATE_COUNT.saturating_sub(self.used()), held.load(Ordering::Acquire)) { - return None; - } + pub(super) fn allocate(&self) -> Option { for _ in 0..64 { let word = self.next.fetch_add(1, Ordering::Relaxed); let index = (word & 63) as usize; @@ -56,119 +29,17 @@ impl WorkStates { .compare_exchange(FREE, word, Ordering::AcqRel, Ordering::Acquire) .is_ok() { - held.fetch_add(1, Ordering::AcqRel); return Some(word); } } None } - pub(super) fn release(&self, word: u32, engine: usize) -> bool { + pub(super) fn release(&self, word: u32) -> bool { let index = (word & 63) as usize; - let released = index != STATE_COUNT + index != STATE_COUNT && self.live[index] .compare_exchange(word, FREE, Ordering::AcqRel, Ordering::Acquire) - .is_ok(); - if released { - if let Some(held) = self.held.get(engine) { - held.fetch_sub(1, Ordering::AcqRel); - } - } - released - } -} - -/// Ordinals past the turn remembered for ioctls that took a node or left early. -pub(super) const ORDER_WINDOW: u64 = 1024; - -/// How far past the oldest waiting ioctl an ioctl may take its node. -pub(super) const RUN_AHEAD: u64 = 8; - -/// The order in which a context's ioctls take work-state nodes. -/// -/// Each engine admits a context's ioctls in submission order, but the engines advance -/// independently: render admission can take nodes for ioctls far ahead of an older ioctl still -/// waiting for its turn on the compute queue. Under deep mixed load (128 compute and 64 render -/// ioctls in flight on one queue) the firmware then stops completing all of the context's work -/// within seconds. Native macOS schedules a queue's command buffers in commit order across -/// engines, at most 36 between scheduling and completion, and the same load runs there without -/// failure. -/// -/// So an ioctl takes its node only while at most [`RUN_AHEAD`] older ioctls of its context have -/// neither taken one nor left without one. Native scheduling happens at commit, on one thread; -/// here each engine admits from its own scheduler, and a strictly ordered turn costs a scheduler -/// round trip whenever the other engine's ioctl is ready first. A short run-ahead avoids that -/// without letting one engine get far ahead of the other. -pub(super) struct Order { - /// The next ordinal to hand out. - next: u64, - /// The oldest ordinal that has neither taken a node nor left. - turn: u64, - /// Ordinals past the turn that took a node or left, as bits at ordinal % ORDER_WINDOW. - done: [u64; (ORDER_WINDOW / 64) as usize], - /// Set once an ioctl left too far past the turn to remember; admission is unordered after. - unordered: bool, - /// An ioctl was refused since the turn last moved: its publisher waits for the next move. - refused: bool, -} -impl Order { - pub(super) const fn new() -> Self { - Self { - next: 0, - turn: 0, - done: [0; (ORDER_WINDOW / 64) as usize], - unordered: false, - refused: false, - } - } - /// The ordinal of a new ioctl, in submission order. - pub(super) fn assign(&mut self) -> u64 { - let ordinal = self.next; - self.next += 1; - ordinal - } - /// Whether the ioctl `ordinal` may take its node now. A refusal is remembered until the - /// turn next moves, so that move wakes the refused publisher. - pub(super) fn admits(&mut self, ordinal: u64) -> bool { - let admits = self.unordered || ordinal <= self.turn + RUN_AHEAD; - self.refused |= !admits; - admits - } - /// The ioctl `ordinal` took its node. Returns whether a refused ioctl may now be admitted. - pub(super) fn admit(&mut self, ordinal: u64) -> bool { - self.finish(ordinal) - } - /// The ioctl `ordinal` left without taking a node (it failed or was cancelled first). - /// Returns whether a refused ioctl may now be admitted. - pub(super) fn leave(&mut self, ordinal: u64) -> bool { - self.finish(ordinal) - } - fn finish(&mut self, ordinal: u64) -> bool { - if self.unordered || ordinal < self.turn { - return false; - } - if ordinal == self.turn { - self.pass(); - return core::mem::take(&mut self.refused); - } - if ordinal - self.turn >= ORDER_WINDOW { - self.unordered = true; - return core::mem::take(&mut self.refused); - } - let bit = ordinal % ORDER_WINDOW; - self.done[(bit / 64) as usize] |= 1 << (bit % 64); - false - } - /// Moves the turn past the current ordinal and every following one already done. - fn pass(&mut self) { - loop { - self.turn += 1; - let bit = self.turn % ORDER_WINDOW; - let (word, mask) = ((bit / 64) as usize, 1u64 << (bit % 64)); - if self.done[word] & mask == 0 { - break; - } - self.done[word] &= !mask; - } + .is_ok() } } @@ -198,131 +69,3 @@ impl RenderSlots { Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) } } - -impl WorkStates { - pub(super) fn used(&self) -> usize { - self.live.iter().filter(|word| word.load(Ordering::Acquire) != FREE).count() - } -} -impl RenderSlots { - pub(super) fn used(&self) -> usize { self.pairs.iter().flatten().count() } -} - -#[kunit_tests(asahi_g17_work_states)] -mod tests { - use super::*; - - #[test] - fn last_node_stays_with_an_engine_holding_none() { - assert!(!may_take(0, 0)); - assert!(may_take(1, 0)); - assert!(!may_take(1, 3)); - assert!(may_take(2, 3)); - } - - #[test] - fn render_cannot_starve_compute_of_a_node() { - let states = WorkStates::new(); - let mut render = KVec::new(); - while let Some(word) = states.allocate(0) { - render.push(word, GFP_KERNEL).unwrap(); - } - // Render stops one short; compute takes the last node, then nothing is left. - assert_eq!(render.len(), STATE_COUNT - 1); - let compute = states.allocate(1).unwrap(); - assert_eq!(states.used(), STATE_COUNT); - assert!(states.allocate(0).is_none()); - assert!(states.allocate(1).is_none()); - // A lone free node goes to the engine holding none. - assert!(states.release(compute, 1)); - assert!(states.allocate(0).is_none()); - let compute = states.allocate(1).unwrap(); - // With both engines holding nodes, a lone free node waits for one of their returns. - assert!(states.release(render.pop().unwrap(), 0)); - assert!(states.allocate(0).is_none()); - assert!(states.allocate(1).is_none()); - // Two free nodes: render may take one again, never the last. - assert!(states.release(render.pop().unwrap(), 0)); - assert!(states.allocate(0).is_some()); - assert!(states.allocate(0).is_none()); - assert!(states.release(compute, 1)); - assert!(states.allocate(1).is_some()); - } - - #[test] - fn ioctls_take_nodes_at_most_run_ahead_past_the_oldest_waiting_one() { - let mut order = Order::new(); - let ordinals: [u64; RUN_AHEAD as usize + 3] = core::array::from_fn(|_| order.assign()); - let (oldest, last) = (ordinals[0], ordinals[RUN_AHEAD as usize + 2]); - // Every ioctl up to RUN_AHEAD past the oldest waiting one may take its node. - for &ordinal in &ordinals[..=RUN_AHEAD as usize] { - assert!(order.admits(ordinal)); - } - assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); - // Ioctls ahead of the turn taking their nodes do not move it. - for &ordinal in &ordinals[1..=RUN_AHEAD as usize] { - assert!(!order.admit(ordinal)); - } - assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); - // The oldest one moves the turn past everything already admitted, and wakes the refused. - assert!(order.admit(oldest)); - assert!(order.admits(last)); - let next = order.assign(); - assert!(order.admits(next)); - assert!(!order.admits(next + RUN_AHEAD + 1)); - } - - #[test] - fn ioctls_leaving_without_a_node_pass_the_turn() { - let mut order = Order::new(); - let (a, b, c) = (order.assign(), order.assign(), order.assign()); - let later: [u64; RUN_AHEAD as usize] = core::array::from_fn(|_| order.assign()); - let beyond = later[RUN_AHEAD as usize - 1]; - // b and c fail before their turn; while a waits, ioctls past a + RUN_AHEAD wait too. - assert!(!order.leave(b)); - assert!(!order.leave(c)); - assert!(!order.admits(beyond)); - // Once a takes its node, the turn passes b and c as well, and the refused one may retry. - assert!(order.admit(a)); - assert!(order.admits(beyond)); - // With nobody refused since, moving the turn wakes nobody; an ordinal already passed - // leaves without effect. - assert!(!order.leave(later[0])); - assert!(!order.leave(a)); - } - - #[test] - fn an_ioctl_leaving_too_far_ahead_unorders_admission() { - let mut order = Order::new(); - let first = order.assign(); - let mut last = first; - for _ in 0..ORDER_WINDOW { - last = order.assign(); - } - assert!(!order.admits(last)); - assert!(order.leave(last)); - assert!(order.admits(last - 1)); - } - - #[test] - fn single_engine_admission_wakes_nobody() { - let mut order = Order::new(); - for _ in 0..3 * RUN_AHEAD { - let ordinal = order.assign(); - assert!(order.admits(ordinal)); - assert!(!order.admit(ordinal)); - } - } - - #[test] - fn releases_return_nodes_to_their_engine() { - let states = WorkStates::new(); - let word = states.allocate(1).unwrap(); - assert!(states.release(word, 1)); - assert!(!states.release(word, 1)); - assert_eq!(states.used(), 0); - assert_eq!(states.held[1].load(Ordering::Acquire), 0); - // An engine outside the table takes nothing. - assert!(states.allocate(ENGINES).is_none()); - } -} diff --git a/drivers/gpu/drm/asahi/g17/data_segment.rs b/drivers/gpu/drm/asahi/g17/data_segment.rs deleted file mode 100644 index 02bcc66de8f51..0000000000000 --- a/drivers/gpu/drm/asahi/g17/data_segment.rs +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -// Copyright (C) 2026 The Gravity Linux Contributors -// -// Derived from the crash buffer mapping of Gravity Linux's M4 GPU driver (CrashBuffer and -// BootOps::shmem_map in drivers/gpu/drm/asahi/g16.rs), written by Cody Ho: -// https://github.com/GravityLinux/linux/commit/bd25d194b0a295a75692a0e6caed3793603b9e65 -// Gravity's g16.rs is adapted from Niklas Sheth's linux-m4-integration prototype. - -//! Crash log buffers in the GPU firmware's data segment. -//! -//! A firmware that places its own crash log ([`super::CrashBuffers::FirmwareData`]) asks for it -//! by physical address inside its data segment, the `fw-data` reserved region, which the boot -//! loader keeps for the loaded firmware until reboot. - -use kernel::{ - c_str, device, - io::{ - self, - IoSysMap, - Region, // - }, - prelude::*, -}; - -/// `size` bytes at `offset` of the firmware's data segment, mapped write-back. -pub(super) struct DataSegmentLog { - mem: io::mem::Mem, - offset: usize, - size: usize, -} - -impl DataSegmentLog { - /// Maps the `size` bytes at physical address `iova` that the firmware asked for, which - /// must lie inside the `fw-data` region of `dev`. - pub(super) fn map(dev: &device::Device, iova: usize, size: usize) -> Result { - let node = dev.of_node().ok_or(EINVAL)?; - let res = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; - let offset = iova.checked_sub(res.start().try_into()?).ok_or(EINVAL)?; - if offset.checked_add(size).ok_or(EINVAL)? > res.size().try_into()? { - return Err(EINVAL); - } - // SAFETY: The boot loader reserved this RAM for the loaded GPU firmware, which keeps - // it until reboot; only the requested subrange is exposed. - let mem = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; - Ok(Self { mem, offset, size }) - } - - /// The CPU view of the buffer. - pub(super) fn buf(&mut self) -> Result> { - // SAFETY: map() checked that the range lies inside the mapping. - let base = unsafe { self.mem.ptr().add(self.offset) }; - let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, self.size)?; - // SAFETY: The range is mapped RAM for as long as this borrow of the mapping. - Ok(IoSysMap::Sys(unsafe { io::SysMem::new(ptr) })) - } -} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 5a0ed611b1dde..5abd494eeb1f5 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -9,7 +9,6 @@ use super::{ use kernel::prelude::*; /// A failed producer does not establish a fault in its consumer's VM. -#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn inherited_error(error: Error) -> Error { if error == EIO || error == ENODEV || error == ETIMEDOUT { ECANCELED @@ -18,61 +17,6 @@ pub(crate) fn inherited_error(error: Error) -> Error { } } -/// What a G17 consumer of a failed producer fails with, whatever the producer failed with: its -/// client does not see a cancellation as a lost device. A producer a recovery dropped fails with -/// ENODATA, and passing that on made every consumer of the producer's buffers lose its device too. -pub(crate) const PRODUCER_FAILED: Error = ECANCELED; - -/// A replay's original host inputs must still be signalled. Treat an unmet -/// input as a failed replay, never as a wait under the device mutex. -pub(crate) fn replay_status(status: i32) -> Result { - if status == 0 { Err(EBUSY) } - else if status < 0 { Err(PRODUCER_FAILED) } - else { Ok(()) } -} - -/// Whether a failed command may still run or write: it holds its VM pin until it has physically -/// retired, or was never published. -pub(crate) trait Retention { - fn retained(&self) -> bool; -} - -/// The failed ordinals of one engine that later commands must not bypass. A command whose prefix -/// covers one is cancelled, so none runs before a failed producer the firmware may still replay. -/// Once that producer has retired nothing of it can run or write any more, so it stops counting: -/// otherwise one cancelled command cancelled every later command of its queue for good. -pub(crate) struct FailureRecord { - retained: KVec<(u64, T)>, - /// A failure that could not be tracked: it keeps cancelling for good, as all failures once did. - untracked: Option<(u64, T)>, -} - -impl FailureRecord { - pub(crate) const fn new() -> Self { - Self { retained: KVec::new(), untracked: None } - } - - /// A finished frontier entry, at pruning. - pub(crate) fn note(&mut self, sequence: u64, status: i32, entry: &T) { - if status >= 0 || !entry.retained() { - return; - } - if self.retained.push((sequence, entry.clone()), GFP_KERNEL).is_err() - && self.untracked.as_ref().is_none_or(|(first, _)| sequence < *first) - { - self.untracked = Some((sequence, entry.clone())); - } - } - - /// The earliest failure later prefixes must not bypass. - pub(crate) fn earliest(&mut self) -> Option<(u64, &T)> { - self.retained.retain(|(_, entry)| entry.retained()); - Iterator::chain(self.retained.iter(), self.untracked.iter()) - .min_by_key(|(sequence, _)| *sequence) - .map(|(sequence, entry)| (*sequence, entry)) - } -} - pub(crate) struct Frontier { dependencies: [KickDependency; KICK_BARRIERS_MAX], sequences: [u64; KICK_BARRIERS_MAX], @@ -137,79 +81,3 @@ impl Frontier { self.insert(sequence, dependency) } } - -#[kunit_tests(asahi_g17_dependency)] -mod tests { - use super::*; - - #[derive(Clone)] - struct Probe(kernel::sync::Arc); - - impl Probe { - fn new(retained: bool) -> Self { - Self(kernel::sync::Arc::new(core::sync::atomic::AtomicBool::new(retained), GFP_KERNEL).unwrap()) - } - fn retire(&self) { - self.0.store(false, core::sync::atomic::Ordering::Relaxed); - } - } - - impl Retention for Probe { - fn retained(&self) -> bool { - self.0.load(core::sync::atomic::Ordering::Relaxed) - } - } - - fn earliest(record: &mut FailureRecord) -> Option { - record.earliest().map(|(sequence, _)| sequence) - } - - #[test] - fn a_cancelled_command_does_not_poison_its_queue() { - // Cancelled or failed before publication: it released its pin and can never run. - let mut record = FailureRecord::new(); - record.note(5, ECANCELED.to_errno(), &Probe::new(false)); - assert_eq!(earliest(&mut record), None); - } - - #[test] - fn a_retained_failure_cancels_until_it_retires() { - let mut record = FailureRecord::new(); - let failed = Probe::new(true); - record.note(5, EIO.to_errno(), &failed); - assert_eq!(earliest(&mut record), Some(5)); - failed.retire(); - assert_eq!(earliest(&mut record), None); - } - - #[test] - fn a_later_retained_failure_survives_an_earlier_retirement() { - let mut record = FailureRecord::new(); - let (first, second) = (Probe::new(true), Probe::new(true)); - record.note(5, EIO.to_errno(), &first); - record.note(9, ECANCELED.to_errno(), &second); - assert_eq!(earliest(&mut record), Some(5)); - first.retire(); - assert_eq!(earliest(&mut record), Some(9)); - second.retire(); - assert_eq!(earliest(&mut record), None); - } - - #[test] - fn successes_are_not_recorded() { - let mut record = FailureRecord::new(); - record.note(5, 1, &Probe::new(true)); - assert_eq!(earliest(&mut record), None); - } - - #[test] - fn failed_predecessor_cannot_be_bypassed() { - let mut frontier = Frontier::new(); - for error in [ECANCELED, EIO, ETIMEDOUT] { - assert_eq!(frontier.include(Some(5), 5, error.to_errno(), 0), Err(ECANCELED)); - } - assert_eq!(frontier.include(Some(5), 5, 1, 0), Ok(())); - assert_eq!(frontier.include(Some(5), 6, EIO.to_errno(), 0), Ok(())); - assert_eq!(frontier.count, 0); - } -} diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index a175e58392379..9e4e0253b7c81 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -11,8 +11,6 @@ use kernel::prelude::*; /// The bounded shared-ring transport; recognized malformed entries return errors after consume. pub(super) trait Transport { - /// The firmware instances of this GPU: the primary, and the secondary where one exists. - fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); fn drain( @@ -36,8 +34,6 @@ pub(super) trait Queues { fn retain_render_completions(&mut self, masks: [u64; 2]); fn complete_compute(&mut self, masks: [u64; 2]) -> Result; fn context_killed(&mut self, cookie: u64); - /// The firmware failed a compute command (see [`Event::ComputeFault`]). - fn compute_fault(&mut self, qid: u8, stamp: u32); } /// Work requested after releasing the device mutex. @@ -74,12 +70,6 @@ fn batch(host: &mut H, role: Role, masks: &mut [u64; 2]) -> Result { } Event::FreeListGrow(completion) => host.complete_freelist_grow(completion)?, Event::ContextKilled(cookie) => host.context_killed(cookie), - Event::ComputeFault { qid, stamp } => { - crate::g17::trace::teardown(1, u64::from(qid), u64::from(stamp), 0); - host.compute_fault(qid, stamp); - // Its queue retires or settles its quarantine on this pass. - masks[usize::from(qid) / 64] |= 1 << (qid % 64); - } Event::Unknown(_) => {} } Ok(true) @@ -91,10 +81,10 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for &role in host.roles() { + for role in [Role::Primary, Role::Secondary] { host.trace(role); } - for &role in host.roles() { + for role in [Role::Primary, Role::Secondary] { host.statistics(role); } let mut masks = [0; 2]; @@ -120,7 +110,7 @@ pub(super) fn service(host: &mut H) -> Result { return Err(EIO); } } - if !host.control_backpressured() && host.roles().contains(&Role::Secondary) { + if !host.control_backpressured() { batch(host, Role::Secondary, &mut masks)?; } host.retain_render_completions(masks); diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index bd1228a82e26f..4cd05fde16581 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -44,8 +44,6 @@ struct Detector { busy: AtomicU64, streak: AtomicU32, until: AtomicU64, - /// Until when the GPU counts as the bottleneck (see `Feed::gpu_bound`). - bound_until: AtomicU64, active: AtomicBool, pending: AtomicBool, stopping: AtomicBool, @@ -59,7 +57,6 @@ impl Detector { busy: AtomicU64::new(0), streak: AtomicU32::new(0), until: AtomicU64::new(0), - bound_until: AtomicU64::new(0), active: AtomicBool::new(false), pending: AtomicBool::new(false), stopping: AtomicBool::new(false), @@ -111,26 +108,17 @@ impl Detector { Err(_) => 0, } }; - if streak < STREAK_ON { - return; - } - if idle_sum.saturating_mul(4) >= busy_sum { + if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { self.until.store(now + LINGER_NS, Ordering::Release); if !self.active.swap(true, Ordering::AcqRel) { self.pending.store(true, Ordering::Release); } - } else { - self.bound_until.store(now + LINGER_NS, Ordering::Release); } } fn wanted(&self, now: u64) -> bool { !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire) } - - fn bound(&self, now: u64) -> bool { - !self.stopping.load(Ordering::Acquire) && now < self.bound_until.load(Ordering::Acquire) - } } #[cfg(CONFIG_CPU_FREQ)] @@ -159,10 +147,6 @@ impl Requests { } } - /// Add a policy to the pinned requests. - /// - /// # Safety - /// /// The caller holds the policy's read or write lock and keeps it alive. /// Policy removal must subsequently pass through this device's notifier. unsafe fn add(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { @@ -195,7 +179,7 @@ impl Requests { this.maxima[index] = (*policy).cpuinfo.max_freq; } this.policies[index] = policy; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } } @@ -209,7 +193,7 @@ impl Requests { let value = if enabled { (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 } else { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 }; if this.applied[index] != value { // SAFETY: This slot is registered, pinned and exclusively locked; @@ -227,7 +211,7 @@ impl Requests { this.applied[index] = value; } } - applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE; + applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } applied } @@ -251,7 +235,7 @@ impl Requests { } this.policies[index] = ptr::null_mut(); this.maxima[index] = 0; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } fn clear(mut self: Pin<&mut Self>) { @@ -380,13 +364,6 @@ impl Feed { } } - /// Process a cpufreq policy notification. - /// - /// # Safety - /// - /// `block` must be this pinned Feed's registered notifier, and `data` must - /// point to the live policy supplied by the cpufreq notifier core. - /// CREATE must hold its write lock; REMOVE must precede policy release. #[cfg(CONFIG_CPU_FREQ)] unsafe extern "C" fn policy_event( block: *mut bindings::notifier_block, @@ -410,35 +387,24 @@ impl Feed { // held and guarantees REMOVE before the core releases its storage. unsafe { requests.as_mut().add(policy) }; } else { - return bindings::NOTIFY_DONE as i32; + return bindings::NOTIFY_DONE as _; } let enabled = feed.detector.wanted(Self::now()); let applied = requests.as_mut().apply(enabled); feed.applied.store(applied, Ordering::Relaxed); - bindings::NOTIFY_OK as i32 + bindings::NOTIFY_OK as _ } fn now() -> u64 { Monotonic::ktime_get() as u64 } - pub(crate) fn note_pass(&self, start: u64, end: u64) { + pub(crate) fn note_render_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); } } - /// Whether the GPU itself is the bottleneck: passes follow each other with refill gaps - /// under a quarter of their busy time, and the feed does not see host-refill-bound work. - /// Lingers like streaming, so a pause of more than a few milliseconds ends it. - pub(crate) fn gpu_bound(&self) -> bool { - if !cfg!(CONFIG_CPU_FREQ) { - return false; - } - let now = Self::now(); - self.detector.bound(now) && !self.detector.wanted(now) - } - /// Called after a firmware event pass, before queuing the single feed work item. pub(crate) fn take_pending(&self) -> bool { self.detector.pending.swap(false, Ordering::AcqRel) @@ -521,51 +487,3 @@ impl PinnedDrop for Feed { self.shutdown(); } } - -#[kernel::macros::kunit_tests(asahi_g17_feed_detector)] -mod detector_tests { - use super::*; - - /// Notes `passes` passes of `busy` ticks, each `gap` ticks after the previous, 1 ms apart. - fn run(detector: &Detector, passes: u64, busy: u64, gap: u64) -> u64 { - let mut start = 1000; - let mut now = 1_000_000; - for _ in 0..passes { - detector.note(start, start + busy, now); - start += busy + gap; - now += 1_000_000; - } - now - } - - #[test] - fn back_to_back_long_passes_are_gpu_bound() { - let detector = Detector::new(); - // 2 ms passes with 40 us refill gaps, as a deep heavy queue or a heavy row at depth 1. - let now = run(&detector, 16, CLOCK_HZ / 500, CLOCK_HZ / 25_000); - assert!(detector.bound(now)); - assert!(!detector.wanted(now)); - assert!(!detector.bound(now + LINGER_NS)); - } - - #[test] - fn refill_gaps_stream_instead() { - let detector = Detector::new(); - // 20 us passes with 60 us refill gaps: bound by host refills. - let now = run(&detector, 16, CLOCK_HZ / 50_000, CLOCK_HZ / 16_667); - assert!(detector.wanted(now)); - assert!(!detector.bound(now)); - } - - #[test] - fn paced_or_short_runs_are_neither() { - let detector = Detector::new(); - // Each pass starts after a pause of at least the refill limit. - let now = run(&detector, 16, CLOCK_HZ / 500, MAX_REFILL_TICKS); - assert!(!detector.bound(now) && !detector.wanted(now)); - // Fewer back-to-back passes than the streak needs. - let detector = Detector::new(); - let now = run(&detector, u64::from(STREAK_ON) - 1, CLOCK_HZ / 500, 0); - assert!(!detector.bound(now) && !detector.wanted(now)); - } -} diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 9015dc82e4076..c9c9ebda7f6bc 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -59,7 +59,7 @@ impl RenderId { } } } - Err(super::capacity::refused(super::capacity::Site::RenderPoolIds)) + Err(ENOSPC) } } impl Drop for RenderId { @@ -69,84 +69,19 @@ impl Drop for RenderId { } } -/// Free-list backing is by far the largest user of the shared lower GPU VA -/// window, and placing it populates every page before the window is searched. -/// After a refused placement, builders waiting for capacity would allocate and -/// free tens of MiB per attempt while nothing in the window changed. A refused -/// size is therefore refused again without allocating until free-list backing -/// is freed, except for one probe per interval across all callers. -static LOWER_FREED: AtomicU64 = AtomicU64::new(0); -/// `LOWER_FREED + 1` at the latest refusal, or zero after a placement fits. -static LOWER_REFUSED: AtomicU64 = AtomicU64::new(0); -static LOWER_REFUSED_SIZE: AtomicU64 = AtomicU64::new(0); -static LOWER_PROBE_AT: AtomicU64 = AtomicU64::new(0); -const LOWER_PROBE_NS: u64 = 100_000_000; - -/// The window refused free-list backing and nothing was freed since. -pub(crate) fn lower_pressure() -> bool { - let refused = LOWER_REFUSED.load(Ordering::Acquire); - refused != 0 && refused == LOWER_FREED.load(Ordering::Acquire).wrapping_add(1) -} - -/// Place free-list backing of `size` bytes unless the same or a smaller size -/// was refused with nothing freed since; `place` allocates and maps it. -fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { - let size = size as u64; - if lower_pressure() && size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { - let now = super::now_ns(); - let at = LOWER_PROBE_AT.load(Ordering::Relaxed); - if now < at - || LOWER_PROBE_AT - .compare_exchange(at, now + LOWER_PROBE_NS, Ordering::AcqRel, Ordering::Relaxed) - .is_err() - { - return Err(ENOSPC); - } - } - match place() { - Err(error) if error == ENOSPC => { - LOWER_REFUSED_SIZE.store(size, Ordering::Relaxed); - LOWER_PROBE_AT.store(super::now_ns() + LOWER_PROBE_NS, Ordering::Relaxed); - LOWER_REFUSED.store( - LOWER_FREED.load(Ordering::Acquire).wrapping_add(1), - Ordering::Release, - ); - Err(error) - } - Ok(placed) => { - if size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { - LOWER_REFUSED.store(0, Ordering::Release); - } - Ok(placed) - } - Err(error) => Err(error), - } -} - pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, } -impl Drop for Pages { - fn drop(&mut self) { - LOWER_FREED.fetch_add(1, Ordering::AcqRel); - } -} impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; - let backing = place_lower(size, || { - alloc.lower( - size, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - ) - })?; - Self::from_backing(backing, blocks, layout) - } - fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { + let mut backing = alloc.lower( + TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + )?; let base = backing.gpu_va(); backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; Ok(Self { @@ -165,7 +100,7 @@ impl Pages { pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } - pub(crate) fn page_list_va(&self) -> u64 { + fn page_list_va(&self) -> u64 { self.backing.gpu_va() } fn run_list_va(&self) -> u64 { @@ -209,17 +144,14 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; - let (backing, [global, client]) = place_lower(size, || { - alloc.lower_with_aliases( - size, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - [render_global, client], - ) - })?; - let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; + let pages = Pages::new(alloc, blocks, RunLayout::Render)?; + let address = pages.page_list_va(); + let global = pages + .backing + .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; + let client = pages + .backing + .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; Ok(Self { _client: client, _global: global, @@ -590,18 +522,6 @@ impl ComputePool { if self.pages.is_some() { return Err(EBUSY); } - self.write_pages(&pages)?; - self.pages = Some(pages); - Ok(()) - } - /// Preserve both sets of pages on failure. The caller owns the consumed - /// release and returns old pages for destruction outside the device mutex. - pub(crate) fn replace_pages(&mut self, pages: &mut Option) -> Result { - self.write_pages(pages.as_ref().ok_or(EINVAL)?)?; - core::mem::swap(&mut self.pages, pages); - Ok(()) - } - fn write_pages(&mut self, pages: &Pages) -> Result { let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { @@ -620,9 +540,9 @@ impl ComputePool { header.unk_54[..8].copy_from_slice(&bytes); self.control.write(0, header)?; fence(Ordering::SeqCst); + self.pages = Some(pages); Ok(()) } - pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { let pages = self.pages.as_mut().ok_or(EAGAIN)?; let address = pages.page_list_va(); @@ -670,18 +590,10 @@ pub(crate) struct Owner { enum LeasePhase { Preparing, Ready, - /// No reference since the given time. The registration stays live so that - /// the owner's next render reuses it without a control-ring release; the - /// release becomes pending when the linger ends. - Lingering(u64), ReleaseSent(u32), Released, } -/// How long an idle registration of an open owner may stay live: the -/// firmware's effort idle hold. -pub(crate) const LINGER_NS: u64 = 16_000_000; - /// References to one prepopulated owner: preparation and two engine commands /// per render. Published releases do not gate reacquisition; their consumption /// witnesses are carried into the successor. @@ -715,14 +627,10 @@ impl Lease { } let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; let same_owner = previous.owner == owner; - // A lingering registration was never released: it continues like an - // overlap of the owner's previous render. let overlapping = same_owner - && match previous.phase { - LeasePhase::Ready => previous.references >= 2 && previous.commands >= 2, - LeasePhase::Lingering(_) => true, - _ => false, - }; + && previous.phase == LeasePhase::Ready + && previous.references >= 2 + && previous.commands >= 2; let reacquire = previous.reacquirable(); if !overlapping && !reacquire { return Err(EBUSY); @@ -784,35 +692,16 @@ impl Lease { Err(EIO) } /// Both engines have retired this render's exact retained submission. - /// With `linger`, a registration left without references stays live - /// instead of making its final release pending. pub(crate) fn complete_render( &mut self, - linger: bool, ) -> Result> { if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { return Err(EINVAL); } self.references -= 2; self.commands -= 2; - if self.references == 0 && linger { - self.phase = LeasePhase::Lingering(super::now_ns().max(1)); - } Ok(self.pending_release()) } - /// The time since which an idle registration lingers, if it does. - pub(crate) fn lingering(&self) -> Option { - match self.phase { - LeasePhase::Lingering(since) => Some(since), - _ => None, - } - } - /// Ends a linger; the final release becomes pending. - pub(crate) fn end_linger(&mut self) { - if matches!(self.phase, LeasePhase::Lingering(_)) { - self.phase = LeasePhase::Ready; - } - } pub(crate) fn pending_release(&self) -> Option { (self.phase == LeasePhase::Ready && self.references == 0).then(|| { super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) @@ -853,30 +742,3 @@ pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Res let ahead = cursor.wrapping_sub(consumer) & (slots - 1); Ok(ahead == 0 || ahead > pending) } - -impl RenderIds { - pub(crate) fn used(&self) -> u32 { - self.used[0].load(Ordering::Acquire).count_ones() - + (self.used[1].load(Ordering::Acquire) & !(1u64 << 63)).count_ones() - } -} - -#[kunit_tests(asahi_g17_render_id_capacity)] -mod capacity_tests { - use super::*; - - #[test] - fn rejected_pool_construction_returns_all_render_ids() { - let ids = Arc::new(RenderIds::new(), GFP_KERNEL).unwrap(); - for _ in 0..8 { - let mut pools = KVec::with_capacity(127, GFP_KERNEL).unwrap(); - for _ in 0..127 { - pools.push(RenderId::new(&ids).unwrap(), GFP_KERNEL).unwrap(); - } - assert_eq!(ids.used(), 127); - assert!(matches!(RenderId::new(&ids), Err(ENOSPC))); - drop(pools); - assert_eq!(ids.used(), 0); - } - } -} diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs index 9d430d50db365..ec6797637bdbe 100644 --- a/drivers/gpu/drm/asahi/g17/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -161,7 +161,7 @@ impl Graph { } let scratch = args.scratch.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; let discard = args.discard.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; - if scratch > u64::from(u32::MAX) || discard > u64::from(u32::MAX) { + if scratch > u32::MAX as u64 || discard > u32::MAX as u64 { return Err(EINVAL); } let stats = offset_va(args.fw_va, (CONTROL + 0x40) as u64)?; @@ -235,7 +235,7 @@ pub(crate) fn scene_registers(index: usize, scratch: u64, metrics: u64) -> Resul .ok_or(EINVAL)? .checked_add((index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE) .ok_or(EOVERFLOW)?; - if scratch > u64::from(u32::MAX) || scratch & 0xf != 0 { + if scratch > u32::MAX as u64 || scratch & 0xf != 0 { return Err(EINVAL); } let metric = offset_va(metrics, (index * 4) as u64)?; diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index 26c6adcf34c83..c4cf6960db296 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -136,9 +136,6 @@ pub(crate) enum Event { TvbGrow(TvbGrowRequest), FreeListGrow(FreeListCompletion), ContextKilled(u64), - /// The firmware failed a compute command of queue `qid` and stores no completion stamp for - /// it: `stamp` is the value that command would have stored. - ComputeFault { qid: u8, stamp: u32 }, Unknown(u32), } @@ -148,8 +145,6 @@ impl EventRecord { const TVB_GROW: u32 = 6; const FREE_LIST_GROW: u32 = 13; const CONTEXT_KILLED: u32 = 12; - const COMPUTE_FAULT: u32 = 7; - const COMPUTE_QIDS: u32 = 128; const DIAGNOSTIC_PAIRS_MAX: u16 = 5; const TVB_SLOTS: u32 = 127; const SUBPIPES: u32 = 64; @@ -165,16 +160,6 @@ impl EventRecord { unsafe { self.words[0] } } - /// Type 7 is a retirement witness only on the firmware whose layout and - /// visibility semantics are qualified. 25G83 must keep consuming it as an - /// unknown event until its own fault contract is established. - pub(crate) fn decode_for(&self, graph: super::variant::InitGraph) -> Result { - if graph == super::variant::InitGraph::V25G83 && self.tag() == Self::COMPUTE_FAULT { - return Ok(Event::Unknown(Self::COMPUTE_FAULT)); - } - self.decode() - } - /// Validates the record's fields without interpreting resource ownership. /// A malformed known event is an error, unlike an unknown tag. pub(crate) fn decode(&self) -> Result { @@ -213,19 +198,6 @@ impl EventRecord { halt_count: record.halt_count, })) } - Self::COMPUTE_FAULT => { - // SAFETY: Every constructor initializes all integer words. - let words = unsafe { self.words }; - // Only compute faults are known to use this type: any other layout stays an - // unknown event, consumed and ignored as before. - if words[3] >= Self::COMPUTE_QIDS || words[4] & 0xff != 0 { - return Ok(Event::Unknown(Self::COMPUTE_FAULT)); - } - Ok(Event::ComputeFault { - qid: words[3] as u8, - stamp: words[4], - }) - } Self::CONTEXT_KILLED => { // SAFETY: Every constructor initializes all integer words. let words = unsafe { self.words }; @@ -526,7 +498,7 @@ impl WorkSlot { data_master: data_master as u32, items_end, qid, - first: u8::from(first), + first: first as u8, }) } } @@ -553,37 +525,3 @@ impl ContextKill { } } } - -#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_event_records))] -mod tests { - use super::*; - - fn record(words: &[u32]) -> EventRecord { - let mut all = [0u32; EVENT_WORDS]; - all[..words.len()].copy_from_slice(words); - EventRecord::from_words(all) - } - - #[test] - fn a_compute_fault_names_its_queue_and_stamp() { - let event = record(&[7, 1, 2, 0x39, 0x300, 0xffff_fc00, 1]).decode(); - assert_eq!(event, Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); - } - - #[test] - fn v25g83_does_not_accept_an_unqualified_fault_witness() { - use super::super::variant::InitGraph; - let fault = record(&[7, 1, 2, 0x39, 0x300]); - assert_eq!(fault.decode_for(InitGraph::G17P), - Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); - assert_eq!(fault.decode_for(InitGraph::V25G83), Ok(Event::Unknown(7))); - assert_eq!(record(&[7, 1, 2, 128, 0x301]).decode_for(InitGraph::V25G83), - Ok(Event::Unknown(7))); - } - - #[test] - fn an_unrecognized_type_7_layout_stays_unknown() { - assert_eq!(record(&[7, 1, 2, 128, 0x300]).decode(), Ok(Event::Unknown(7))); - assert_eq!(record(&[7, 1, 2, 9, 0x301]).decode(), Ok(Event::Unknown(7))); - } -} diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index b1afdad816b6b..0633da7f43541 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -12,7 +12,7 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; use super::queue::CommandTag; use super::{ - offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, }; @@ -67,10 +67,10 @@ impl ScratchRequest { /// guard; the per-register words are the native formulas, so they are /// checked here against Apple's own values before they reach hardware. pub(crate) fn new(layout: u64) -> Result { - let depth = layout & u64::from(u16::MAX); - let cliques = (layout >> 24) & u64::from(u8::MAX); - let user = (layout >> 32) & u64::from(u16::MAX); - let driver = (layout >> 48) & u64::from(u16::MAX); + let depth = layout & u16::MAX as u64; + let cliques = (layout >> 24) & u8::MAX as u64; + let user = (layout >> 32) & u16::MAX as u64; + let driver = (layout >> 48) & u16::MAX as u64; if user == 0 && driver == 0 { return Ok(Self { pages: 0, @@ -152,42 +152,13 @@ const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; /// Offsets in the preempt buffer of the four state words the program names. const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; -/// Layout of a queue's operand-state buffer, which depends on the GPU's core layout: offsets of -/// the regions the program names (two preemption banks of three 0x800-byte records per core, -/// tagged | 5 at their start, then their end, then the per-cluster tails) and its size. -pub(crate) struct OperandState { - r10229: u64, - r140a8: u64, - r10099: u64, - r10091: u64, - r0a5c1: u64, - r0a5c9: u64, - /// Size of the buffer. - pub(crate) size: usize, -} - -impl OperandState { - /// One cluster of six cores (A18 Pro). - pub(crate) const ONE_BY_SIX: OperandState = OperandState { - r10229: 0x12800, - r140a8: 0x13000, - r10099: 0x9405, - r10091: 0x12400, - r0a5c1: 0x0005, - r0a5c9: 0x9000, - size: 0x14000, - }; - /// Two clusters of ten cores (M4 Pro): bank stride 0x1e400, tails 0x1000. - pub(crate) const TWO_BY_TEN: OperandState = OperandState { - r10229: 0x3c800, - r140a8: 0x3d800, - r10099: 0x1e405, - r10091: 0x3c400, - r0a5c1: 0x0005, - r0a5c9: 0x1e000, - size: 0x44000, - }; -} +/// Offsets in the operand-state buffer of the regions the program names. +const OPERAND_STATE_10229: u64 = 0x12800; +const OPERAND_STATE_140A8: u64 = 0x13000; +const OPERAND_STATE_10099: u64 = 0x9405; +const OPERAND_STATE_10091: u64 = 0x12400; +const OPERAND_STATE_0A5C1: u64 = 0x0005; +const OPERAND_STATE_0A5C9: u64 = 0x9000; /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full @@ -312,8 +283,6 @@ pub(crate) struct ComputeArgs { pub(crate) preempt_va: u64, /// GPU address of the operand-state buffer, as seen by the work's VM. pub(crate) operand_state_va: u64, - /// Layout of the operand-state buffer. - pub(crate) operand: &'static OperandState, /// GPU address of the private-memory usage record, as seen by the work's /// VM. pub(crate) usage_va: u64, @@ -336,7 +305,6 @@ pub(crate) struct ComputeArgs { fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; - let state = args.operand; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); let intersection = args.intersection.words(); @@ -379,14 +347,14 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x0a351, ids), RegisterWrite::new(0x10201, key), RegisterWrite::new(0x10428, key), - RegisterWrite::new(0x14028, u64::from(args.free_list_slot)), + RegisterWrite::new(0x14028, args.free_list_slot as u64), RegisterWrite::new(0x14070, args.usage_va | 1), - RegisterWrite::new(0x10229, offset_va(operand, state.r10229)?), - RegisterWrite::new(0x140a8, offset_va(operand, state.r140a8)?), - RegisterWrite::new(0x10099, offset_va(operand, state.r10099)?), - RegisterWrite::new(0x10091, offset_va(operand, state.r10091)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, state.r0a5c1)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, state.r0a5c9)?), + RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), + RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), + RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), RegisterWrite::new(0x1a440, UNK_1A440_FINAL), RegisterWrite::new(0x0a599, UNK_0A599), ]) @@ -396,12 +364,11 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS /// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { let operand = args.operand_state_va; - let state = args.operand; Ok([ - RegisterWrite::new(0x10099, offset_va(operand, state.r0a5c1)?), - RegisterWrite::new(0x10091, offset_va(operand, state.r0a5c9)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, state.r10099)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, state.r10091)?), + RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), + RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), ]) } @@ -507,9 +474,8 @@ impl ComputeDescriptor { const UNK_FBA: u32 = 0xe0a0_0001; const UNK_FC5: u8 = 0x9f; - /// Fills a descriptor cleared by `Object::initialize`. The unused firmware - /// fields and padding retain that zero initialization. - pub(crate) fn write_zeroed(&mut self, args: &ComputeArgs) -> Result { + /// Writes the complete descriptor. + pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { if args.cdm_end_va < args.cdm_va || (args.sampler_heap_va == 0) != (args.sampler_count == 0) || args.sampler_heap_va & 7 != 0 @@ -523,6 +489,7 @@ impl ComputeDescriptor { let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; let sampler_max = sampler_max(args.sampler_count)?; + clear(self); self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 3092e599caa84..1f25bd0fc1c0d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -27,8 +27,6 @@ use kernel::prelude::*; use super::clear; -pub(crate) mod v25g83; - /// Interface version at root `+0x00`; the firmware rejects any other. pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; @@ -221,21 +219,6 @@ impl SecondaryRoot { config_va, } } - - /// Returns the root of a single instance that carries both roles: the primary root, with - /// status B, extended by the power-state and configuration pointers. - pub(crate) fn merged( - args: &RootArgs, - status_b_va: u64, - power_state_va: u64, - config_va: u64, - ) -> Self { - Self { - root: Root::new(Role::Primary, args, status_b_va), - power_state_va, - config_va, - } - } } /// Offset of the secondary root from the primary root. Both live in one @@ -608,42 +591,31 @@ pub(crate) struct RegisterWindow { pub(crate) phys: u64, /// GPU address of the window in the firmware MMIO aperture. pub(crate) va: u64, - /// Size in bytes of one instance; need not be a multiple of the page size. + /// Size in bytes; need not be a multiple of the page size. pub(crate) size: u32, /// Value of [`RegisterMapEntry::unk_18`]. pub(crate) unk_18: u64, /// Value of [`RegisterMapEntry::unk_20`]. pub(crate) unk_20: u32, - /// Number of instances of the block. The firmware finds instance `n` at - /// `va + n * size`; it lives at `phys + n * phys_stride`. - pub(crate) instances: u32, - /// Distance between the physical addresses of consecutive instances. - pub(crate) phys_stride: u64, } impl RegisterWindow { - /// Returns the page-aligned physical address, GPU address and length that - /// instance `instance` of the window must be mapped with, or `None` if its - /// physical and GPU addresses differ within a page or the mapped span - /// overflows. - pub(crate) fn mapping(&self, instance: u32) -> Option<(u64, u64, u64)> { + /// Returns the page-aligned physical address, GPU address and length the + /// window must be mapped with, or `None` if its physical and GPU + /// addresses differ within a page or the mapped span overflows. + pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { let mask = PAGE_SIZE as u64 - 1; - let phys = self - .phys - .checked_add(self.phys_stride.checked_mul(u64::from(instance))?)?; - let va = self - .va - .checked_add(u64::from(self.size).checked_mul(u64::from(instance))?)?; - if phys & mask != va & mask { + if self.phys & mask != self.va & mask { return None; } - let start = va & !mask; - let span = va - .checked_add(u64::from(self.size))? + let start = self.va & !mask; + let span = self + .va + .checked_add(self.size as u64)? .checked_sub(start)? .checked_add(mask)? & !mask; - Some((phys & !mask, start, span)) + Some((self.phys & !mask, start, span)) } } @@ -657,9 +629,9 @@ pub(crate) struct RegisterMapEntry { pub(crate) phys: u64, /// GPU address of the window. pub(crate) va: u64, - /// Size in bytes of all instances of the window. + /// Window size in bytes. pub(crate) total_size: u32, - /// Size in bytes of one instance. + /// Window size in bytes, again. pub(crate) element_size: u32, /// Per-window value, see [`RegisterWindow::unk_18`]. pub(crate) unk_18: u64, @@ -678,7 +650,7 @@ impl RegisterMapEntry { Self { phys: window.phys, va: window.va, - total_size: window.size * window.instances, + total_size: window.size, element_size: window.size, unk_18: window.unk_18, unk_20: window.unk_20, @@ -704,19 +676,14 @@ pub(crate) const PERF_STATES: usize = 11; /// Voltage lanes of a performance table. const PERF_LANES: usize = 16; -/// Most voltage cells of an operating point: one per GPU cluster, at most one per lane. -pub(crate) const OPP_CELLS: usize = PERF_LANES; - /// One GPU operating point, from the device tree, in performance-state /// order. #[derive(Copy, Clone, Debug)] pub(crate) struct Opp { /// Frequency in Hz. pub(crate) freq_hz: u64, - /// GPU core voltage in µV: one cell for the whole GPU, or one per cluster. - pub(crate) volt_uv: [u32; OPP_CELLS], - /// Number of valid cells in [`Self::volt_uv`]. - pub(crate) cells: usize, + /// GPU core voltage in µV. + pub(crate) volt_uv: u32, } /// Per-SoC parts of the performance tables that the operating points do not @@ -761,23 +728,20 @@ static_assert!(size_of::() == 0x704); #[derive(Copy, Clone)] struct PerfState { freq_mhz: u32, - core_mv: [u32; OPP_CELLS], - sram_mv: [u32; OPP_CELLS], - cells: usize, + core_mv: u32, + sram_mv: u32, } impl PerfGroup { - /// Fills the table. With one voltage cell, `lanes` is the number of voltage lanes - /// holding each state's voltages; with one cell per cluster, lane `n` holds cluster - /// `n`'s. The remaining lanes stay zero. + /// Fills the table. `lanes` is the number of voltage lanes holding each + /// state's voltages; the remaining lanes stay zero. fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { self.unk_0 = 10; for (index, state) in states.iter().enumerate() { self.freq_mhz[index] = state.freq_mhz; - for lane in 0..if state.cells == 1 { lanes } else { state.cells } { - let cell = if state.cells == 1 { 0 } else { lane }; - self.core_mv[index][lane] = state.core_mv[cell]; - self.sram_mv[index][lane] = state.sram_mv[cell]; + for lane in 0..lanes { + self.core_mv[index][lane] = state.core_mv; + self.sram_mv[index][lane] = state.sram_mv; } } } @@ -791,18 +755,16 @@ fn perf_states( ) -> Result<[PerfState; PERF_STATES]> { let mut states = [PerfState { freq_mhz: 0, - core_mv: [0; OPP_CELLS], - sram_mv: [0; OPP_CELLS], - cells: 0, + core_mv: 0, + sram_mv: 0, }; PERF_STATES]; for (state, &index) in states.iter_mut().zip(map) { let opp = opps.get(index as usize).ok_or(EINVAL)?; - let core_mv = opp.volt_uv.map(|uv| uv / 1000); + let core_mv = opp.volt_uv / 1000; *state = PerfState { freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, core_mv, - sram_mv: core_mv.map(|mv| mv.max(sram_min_mv)), - cells: opp.cells, + sram_mv: core_mv.max(sram_min_mv), }; } Ok(states) @@ -1207,35 +1169,6 @@ pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; /// scheduling pass. pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; -/// The values of region C that differ between SoCs and firmware builds. -pub(crate) struct RegionCConfig { - /// A18 Pro: 0xffff. - pub(crate) unk_54: u16, - /// A18 Pro: 1 in every entry. - pub(crate) unk_78: [u32; 3], - /// Highest GPU power, in mW (A18 Pro: 9550). - pub(crate) max_power_mw: u32, - /// A18 Pro: 0x3000_001d. - pub(crate) unk_9c: u32, - /// A18 Pro: 0x3100_0002. - pub(crate) unk_a0: u32, - /// A18 Pro: 0x7d0 (2000). - pub(crate) unk_7c4: u32, - /// A18 Pro: zero. - pub(crate) unk_9e0: [u32; 8], -} - -/// Region C of the A18 Pro firmware (26A428). -pub(crate) const REGION_C_26A428: RegionCConfig = RegionCConfig { - unk_54: 0xffff, - unk_78: [1; 3], - max_power_mw: 9550, - unk_9c: 0x3000_001d, - unk_a0: 0x3100_0002, - unk_7c4: 0x7d0, - unk_9e0: [0; 8], -}; - /// Region C: firmware policy settings shared by both instances. /// /// Three fields are live. The host updates [`Self::effort`] while it submits @@ -1264,7 +1197,7 @@ pub(crate) struct RegionC { /// Required: 0x78 (120). pub(crate) unk_3c: u32, pub(crate) unk_40: [u8; 0x14], - /// See [`RegionCConfig::unk_54`]. + /// Required: 0xffff. pub(crate) unk_54: u16, /// Required: 0x28 (40). pub(crate) unk_56: u16, @@ -1280,10 +1213,10 @@ pub(crate) struct RegionC { /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). pub(crate) context_switch_timer_multiplier: u8, pub(crate) unk_6b: [u8; 0xd], - /// See [`RegionCConfig::unk_78`]. + /// Required: 1 in every entry. pub(crate) unk_78: [u32; 3], - /// Highest GPU power, in mW. - pub(crate) max_power_mw: u32, + /// Required: 0x254e (9550). + pub(crate) unk_84: u32, /// Required: 0x3e8 (1000). pub(crate) unk_88: u32, /// Required: 0x3e8 (1000). @@ -1291,9 +1224,9 @@ pub(crate) struct RegionC { pub(crate) unk_90: [u8; 8], /// Required: 0x3e8 (1000). pub(crate) unk_98: u32, - /// See [`RegionCConfig::unk_9c`]. + /// Required: 0x3000_001d. pub(crate) unk_9c: u32, - /// See [`RegionCConfig::unk_a0`]. + /// Required: 0x3100_0002. pub(crate) unk_a0: u32, pub(crate) unk_a4: [u8; 0x40], /// Performance-floor request (live, see the type documentation). @@ -1301,7 +1234,7 @@ pub(crate) struct RegionC { /// Live; the host stores zero here whenever it stores [`Self::effort`]. pub(crate) unk_e8: u32, pub(crate) unk_ec: [u8; 0x6d8], - /// See [`RegionCConfig::unk_7c4`]. + /// Required: 0x7d0 (2000). pub(crate) unk_7c4: u32, /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). pub(crate) unk_7c8: u32, @@ -1352,10 +1285,7 @@ pub(crate) struct RegionC { pub(crate) compute_watchdog: u32, /// Required: 1. pub(crate) unk_9d0: u32, - pub(crate) unk_9d4: [u8; 0xc], - /// See [`RegionCConfig::unk_9e0`]. - pub(crate) unk_9e0: [u32; 8], - pub(crate) unk_a00: [u8; 0x41d], + pub(crate) unk_9d4: [u8; 0x449], /// Required: 1. pub(crate) unk_e1d: u32, pub(crate) unk_e21: [u8; 0x18], @@ -1389,28 +1319,28 @@ impl RegionC { pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); /// Initializes region C. - pub(crate) fn init(&mut self, cfg: &RegionCConfig) { + pub(crate) fn init(&mut self) { clear(self); self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; self.unk_24 = 0xbb8; self.idle_power_off = 1; self.unk_34 = 1; self.unk_3c = 0x78; - self.unk_54 = cfg.unk_54; + self.unk_54 = 0xffff; self.unk_56 = 0x28; self.unk_58 = 0xffff; self.unk_5e = 1; self.unk_62 = 1; self.unk_66 = 3; self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; - self.unk_78 = cfg.unk_78; - self.max_power_mw = cfg.max_power_mw; + self.unk_78 = [1; 3]; + self.unk_84 = 0x254e; self.unk_88 = 0x3e8; self.unk_8c = 0x3e8; self.unk_98 = 0x3e8; - self.unk_9c = cfg.unk_9c; - self.unk_a0 = cfg.unk_a0; - self.unk_7c4 = cfg.unk_7c4; + self.unk_9c = 0x3000_001d; + self.unk_a0 = 0x3100_0002; + self.unk_7c4 = 0x7d0; self.unk_7c8 = 0x3f80_0000; self.unk_7cc = 0x3f4c_cccd; self.unk_7d0 = 0x3e4c_cccd; @@ -1434,7 +1364,6 @@ impl RegionC { self.unk_9c8 = 0x28; self.compute_watchdog = 50; self.unk_9d0 = 1; - self.unk_9e0 = cfg.unk_9e0; self.unk_e1d = 1; self.unk_e39 = 3; self.unk_e3d = 1; @@ -1482,7 +1411,7 @@ impl StatusBlock { /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; /// Offset of the firmware-control ring in its page. -pub(crate) const FWCTL_RING: u64 = 0x40; +const FWCTL_RING: u64 = 0x40; /// One firmware-written recovery-info entry. The host reads each word separately after the /// recovery handshake reaches state 3; it never copies the live table as a structure. diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs deleted file mode 100644 index f4c3ee5db22e2..0000000000000 --- a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs +++ /dev/null @@ -1,619 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only - -//! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). -//! -//! The M4 Pro (T6040) runs this build. It has the interface version of the A18 Pro's -//! 26A428 build and the same objects, with these differences: -//! -//! - [`HwData25G83`]: the A18 Pro's hardware data up to `0x2554`, eight bytes inserted before -//! `0x2570` and four more before `0x2614`, and the CS and AFR performance tables; -//! - [`PowerConfig25G83`]: the power configuration with two blocks removed (`0xa8` and `0xa0` -//! bytes), at bundle `+0x5380`, followed by the auxiliary data; -//! - region C and the power state ([`POWER_STATE_WORDS`]) with other values; -//! - one firmware instance carrying both roles, with a larger status B ([`private`]), five -//! separately allocated views and a firmware area ([`kernel_window`]). -//! -//! The constant contents were derived from the G16 encoders of Gravity Linux, which run this -//! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. - -use super::{ - fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, - RegisterMapEntry, Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, - COMPLETION_RING_INDEX, PERF_STATES, REGISTER_MAP_SLOTS, -}; -use kernel::prelude::*; - -/// Kernel-window objects whose placement differs from the A18 Pro's, as offsets in the upper -/// kernel window. -pub(crate) mod kernel_window { - /// The power-state object named by the root (`0x6a8` bytes). - pub(crate) const POWER_STATE: u64 = 0x17_0000; - /// The five views named by the main configuration. - pub(crate) const VIEWS: [u64; 5] = [0x41_4000, 0x40_0000, 0x40_4000, 0x40_8000, 0x40_c000]; - /// The firmware area named twice by the main configuration (`0x88` bytes). - pub(crate) const FIRMWARE_AREA: u64 = 0x41_0000; - /// The AON temperature-source table named by the power configuration. - pub(crate) const AON_TABLE: u64 = 0xc050_0000; -} - -/// Offsets in the private cluster. -pub(crate) mod private { - /// Size of the private cluster: the state grid at `+0`, status B and status A. - pub(crate) const SIZE: usize = 0xd4000; - /// The primary status B ([`super::super::PrimaryStatusB`] up to its power state). - pub(crate) const STATUS_B: usize = 0x1_0000; - /// The primary status A, after the `0xf180` bytes of status B. - pub(crate) const STATUS_A: usize = 0x1_f180; - /// The configuration named by the root; its status follows at `+0x80`. - pub(crate) const CONFIG: usize = 0xd_3440; - /// The zeroed page named by the main configuration's `unk_18`. - pub(crate) const MAIN_HEADER: usize = 0x8000; -} - -/// Bundle offset of the [`PowerConfig25G83`]. -pub(crate) const POWER_CONFIG: usize = 0x5380; -/// Bundle offset of the completion-ring descriptors in the [`HwData25G83`]. -pub(crate) const COMPLETION_DESCRIPTORS: usize = 0x2620; -/// Bundle offset of the QoS pointer in the [`HwData25G83`]. -pub(crate) const QOS_POINTER: usize = 0x26b8; - -/// Most states of a CS or AFR performance table. -const AUX_PERF_STATES: usize = 16; - -/// The CS or AFR performance table of the hardware data. -#[repr(C, packed)] -pub(crate) struct AuxPerfTable { - /// Highest state, twice. - pub(crate) max_state: [u32; 2], - /// Frequency of each state in MHz. - pub(crate) freq_mhz: [u32; AUX_PERF_STATES], - /// Voltage of each state in mV, in the first word of each pair. - pub(crate) mv: [[u32; 2]; AUX_PERF_STATES], - /// SRAM voltage of each state in mV, in the first word of each pair. - pub(crate) sram_mv: [[u32; 2]; AUX_PERF_STATES], -} - -// SAFETY: `AuxPerfTable` consists of integers and arrays of integers only. -unsafe impl Zeroable for AuxPerfTable {} - -static_assert!(size_of::() == 0x148); - -impl AuxPerfTable { - /// Fills the table from its operating points (one voltage cell each), raising each SRAM - /// voltage to `sram_min_mv`. - fn fill(&mut self, opps: &[Opp], sram_min_mv: u32) -> Result { - if !(2..=AUX_PERF_STATES).contains(&opps.len()) { - return Err(EINVAL); - } - self.max_state = [opps.len() as u32 - 1; 2]; - for (index, opp) in opps.iter().enumerate() { - if opp.cells != 1 { - return Err(EINVAL); - } - let mv = opp.volt_uv[0] / 1000; - self.freq_mhz[index] = u32::try_from(opp.freq_hz / 1_000_000)?; - self.mv[index][0] = mv; - self.sram_mv[index][0] = mv.max(sram_min_mv); - } - Ok(()) - } -} - -/// Inputs of the 25G83 hardware data in addition to [`HwDataArgs`]. -pub(crate) struct HwDataArgs25G83<'a> { - /// Enabled GPU cores. - pub(crate) active_cores: u32, - /// Operating points of the CS clock domain. - pub(crate) cs_opps: &'a [Opp], - /// Operating points of the AFR clock domain. - pub(crate) afr_opps: &'a [Opp], -} - -/// The hardware data of the 25G83 build, at the base of the bundle. -#[repr(C, packed)] -pub(crate) struct HwData25G83 { - /// Required: 0x6f_0000_0000. - pub(crate) unk_0: u64, - /// Required: 0xffc0_0000. - pub(crate) unk_8: u64, - /// Required: 0x10_0000_0000. - pub(crate) unk_10: u64, - /// Required: 0x10_0000_0000. - pub(crate) unk_18: u64, - /// Required: 0x2ff_ffff_8000. - pub(crate) unk_20: u64, - /// Required: 0xffff_fc21_8140_0000. - pub(crate) unk_28: u64, - pub(crate) unk_30: [u8; 0x610], - /// Register windows the firmware maps. - pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], - pub(crate) unk_e88: [u8; 8], - /// Chip identifier. - pub(crate) chip_id: u32, - /// Required: 1. - pub(crate) unk_e94: u32, - /// Required: 1. - pub(crate) unk_e98: u32, - pub(crate) unk_e9c: [u8; 4], - /// Required: 1. - pub(crate) unk_ea0: u32, - /// Required: 1. - pub(crate) unk_ea4: u32, - pub(crate) unk_ea8: [u8; 0x10], - /// Required: 1. - pub(crate) unk_eb8: u32, - pub(crate) unk_ebc: [u8; 4], - /// Required: 1. - pub(crate) unk_ec0: u32, - pub(crate) unk_ec4: [u8; 4], - /// Required: 1. - pub(crate) unk_ec8: u32, - pub(crate) unk_ecc: [u8; 4], - /// Required: 0x5dc0 (24000). - pub(crate) unk_ed0: u32, - /// Required: 1. - pub(crate) unk_ed4: u32, - /// Power sample period in ms: 8. - pub(crate) unk_ed8: u32, - pub(crate) unk_edc: [u8; 4], - /// Required: 1 in every entry. - pub(crate) unk_ee0: [u32; 3], - pub(crate) unk_eec: [u8; 0x18], - /// Required: 0x1f (31). - pub(crate) unk_f04: u32, - pub(crate) unk_f08: [u8; 0x1c], - /// Required: 0x0700_2004. - pub(crate) unk_f24: u32, - /// Required: 6. - pub(crate) unk_f28: u32, - pub(crate) unk_f2c: [u8; 8], - /// Required: 1. - pub(crate) unk_f34: u32, - /// Required: 1. - pub(crate) unk_f38: u32, - pub(crate) unk_f3c: [u8; 0x10], - /// Required: 0x31 (49). - pub(crate) unk_f4c: u32, - pub(crate) unk_f50: [u8; 0x38], - /// Enabled GPU cores. - pub(crate) unk_f88: u32, - /// Required: 0xf (15). - pub(crate) unk_f8c: u32, - pub(crate) unk_f90: [u8; 0x28], - /// Required: 0x1c (28). - pub(crate) unk_fb8: u32, - /// Required: 4. - pub(crate) unk_fbc: u32, - /// Enabled GPU cores. - pub(crate) unk_fc0: u32, - /// Performance table A, one voltage lane per cluster. - pub(crate) perf_a: PerfGroup, - pub(crate) unk_16c8: [u8; 0x140], - /// Frequency of each state of table B in MHz. - pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], - pub(crate) unk_1834: [u8; 0x14], - /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. - pub(crate) unk_1848: [u32; 11], - pub(crate) unk_1874: [u8; 0x54], - /// Relative performance of each state of table A, in percent. - pub(crate) perf_a_relative: [u32; PERF_STATES], - pub(crate) unk_18f4: [u8; 0x14], - /// Relative performance of each state of table B, in percent. - pub(crate) perf_b_relative: [u32; PERF_STATES], - pub(crate) unk_1934: [u8; 0x94], - /// Operating point of each state of table A. - pub(crate) perf_a_states: [u32; PERF_STATES], - pub(crate) unk_19f4: [u8; 0x14], - /// Operating point of each state of table B. - pub(crate) perf_b_states: [u32; PERF_STATES], - pub(crate) unk_1a34: [u8; 0x14], - /// CS clock-domain performance table. - pub(crate) cs_perf: AuxPerfTable, - /// AFR clock-domain performance table. - pub(crate) afr_perf: AuxPerfTable, - /// Performance table B, one voltage lane per cluster. - pub(crate) perf_b: PerfGroup, - pub(crate) unk_23dc: [u8; 0x168], - /// Required: 5. - pub(crate) unk_2544: u32, - /// Required: 6. - pub(crate) unk_2548: u32, - /// Required: 3. - pub(crate) unk_254c: u32, - /// Required: 7. - pub(crate) unk_2550: u32, - /// Required: 7. - pub(crate) unk_2554: u32, - pub(crate) unk_2558: [u8; 0x18], - /// Enabled GPU cores. - pub(crate) unk_2570: u32, - pub(crate) unk_2574: [u8; 4], - /// Required: 1. - pub(crate) unk_2578: u32, - pub(crate) unk_257c: [u8; 0x1c], - /// Required: 0, 1, 1, 1. - pub(crate) unk_2598: [u32; 4], - /// Required: 0, 1, 1, 1. - pub(crate) unk_25a8: [u32; 4], - pub(crate) unk_25b8: [u8; 4], - /// Required: 0xffff_ffff in every entry, one per work channel. - pub(crate) unk_25bc: [u32; 12], - pub(crate) unk_25ec: [u8; 8], - /// Required: 2. - pub(crate) unk_25f4: u32, - pub(crate) unk_25f8: [u8; 4], - /// Required: 1. - pub(crate) unk_25fc: u32, - pub(crate) unk_2600: [u8; 0x14], - /// Required: 1. - pub(crate) unk_2614: u32, - pub(crate) unk_2618: [u8; 4], - /// Required: 0x100. - pub(crate) unk_261c: u32, - /// Completion-ring descriptors. - pub(crate) completion_rings: [CompletionRingDescriptor; 4], - pub(crate) unk_26a0: [u8; 0x18], - /// QoS object, zeroed. - pub(crate) qos_va: u64, - /// Required: 1. - pub(crate) unk_26c0: u32, - /// Required: 1. - pub(crate) unk_26c4: u32, - pub(crate) unk_26c8: [u8; 0x2c], - /// Required: 1. - pub(crate) unk_26f4: u32, - pub(crate) unk_26f8: [u8; 0x16bc], -} - -static_assert!(core::mem::offset_of!(HwData25G83, register_map) == 0x640); -static_assert!(core::mem::offset_of!(HwData25G83, chip_id) == 0xe90); -static_assert!(core::mem::offset_of!(HwData25G83, perf_a) == 0xfc4); -static_assert!(core::mem::offset_of!(HwData25G83, perf_b_freq_mhz) == 0x1808); -static_assert!(core::mem::offset_of!(HwData25G83, cs_perf) == 0x1a48); -static_assert!(core::mem::offset_of!(HwData25G83, afr_perf) == 0x1b90); -static_assert!(core::mem::offset_of!(HwData25G83, perf_b) == 0x1cd8); -static_assert!(core::mem::offset_of!(HwData25G83, unk_2570) == 0x2570); -static_assert!(core::mem::offset_of!(HwData25G83, completion_rings) == COMPLETION_DESCRIPTORS); -static_assert!(core::mem::offset_of!(HwData25G83, qos_va) == QOS_POINTER); -static_assert!(size_of::() == 0x3db4); -static_assert!(size_of::() <= POWER_CONFIG); - -// SAFETY: `HwData25G83` consists of integers and arrays of integers only. -unsafe impl Zeroable for HwData25G83 {} - -impl HwData25G83 { - /// Initializes the hardware data. Table A and B carry one voltage lane per cluster. - pub(crate) fn init(&mut self, args: &HwDataArgs<'_>, ext: &HwDataArgs25G83<'_>) -> Result { - super::clear(self); - self.unk_0 = 0x6f_0000_0000; - self.unk_8 = 0xffc0_0000; - self.unk_10 = 0x10_0000_0000; - self.unk_18 = 0x10_0000_0000; - self.unk_20 = 0x2ff_ffff_8000; - self.unk_28 = 0xffff_fc21_8140_0000; - - let mut used = [false; REGISTER_MAP_SLOTS]; - for window in args.windows { - match used.get_mut(window.slot) { - Some(used) if !*used => *used = true, - _ => return Err(EINVAL), - } - self.register_map[window.slot] = RegisterMapEntry::window(window); - } - if !args.reserved_slots.is_empty() { - return Err(EINVAL); - } - - self.chip_id = args.chip_id; - self.unk_e94 = 1; - self.unk_e98 = 1; - self.unk_ea0 = 1; - self.unk_ea4 = 1; - self.unk_eb8 = 1; - self.unk_ec0 = 1; - self.unk_ec8 = 1; - self.unk_ed0 = 0x5dc0; - self.unk_ed4 = 1; - self.unk_ed8 = 8; - self.unk_ee0 = [1; 3]; - self.unk_f04 = 0x1f; - self.unk_f24 = 0x0700_2004; - self.unk_f28 = 6; - self.unk_f34 = 1; - self.unk_f38 = 1; - self.unk_f4c = 0x31; - self.unk_f88 = ext.active_cores; - self.unk_f8c = 0xf; - self.unk_fb8 = 0x1c; - self.unk_fbc = 4; - self.unk_fc0 = ext.active_cores; - - let perf = args.perf; - let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; - let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; - if states_a[0].cells < 2 { - return Err(EINVAL); - } - self.perf_a.fill(&states_a, 1); - self.perf_b.fill(&states_b, 1); - for (index, state) in states_b.iter().enumerate() { - self.perf_b_freq_mhz[index] = state.freq_mhz; - self.perf_a_states[index] = perf.states_a[index].into(); - self.perf_b_states[index] = perf.states_b[index].into(); - } - self.perf_a_relative = perf.relative_a; - self.perf_b_relative = perf.relative_b; - self.unk_1848 = [0x3f82_8f5c; 11]; - self.cs_perf.fill(ext.cs_opps, perf.sram_min_mv)?; - self.afr_perf.fill(ext.afr_opps, perf.sram_min_mv)?; - - self.unk_2544 = 5; - self.unk_2548 = 6; - self.unk_254c = 3; - self.unk_2550 = 7; - self.unk_2554 = 7; - self.unk_2570 = ext.active_cores; - self.unk_2578 = 1; - self.unk_2598 = [0, 1, 1, 1]; - self.unk_25a8 = [0, 1, 1, 1]; - self.unk_25bc = [u32::MAX; 12]; - self.unk_25f4 = 2; - self.unk_25fc = 1; - self.unk_2614 = 1; - self.unk_261c = 0x100; - for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { - self.completion_rings[index as usize] = CompletionRingDescriptor { - low_va: ring.low_va, - high_va: ring.high_va, - entries: COMPLETION_RING_ENTRIES, - entry_size: COMPLETION_RING_ENTRY_SIZE, - index, - unk_1c: 0, - }; - } - self.qos_va = args.qos_va; - self.unk_26c0 = 1; - self.unk_26c4 = 1; - self.unk_26f4 = 1; - Ok(()) - } -} - -/// Power configuration control period, in 24 MHz clocks (8 ms). -const PERIOD_CLOCKS: u32 = 200_000; -/// Temperature sensors read by the die and MTR temperature limiters. -const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; - -/// Offset of the per-chip leakage coefficients in the power configuration. -const LEAKAGE_OFFSET: usize = 0x642c; -/// Offset in the power configuration of the GPU address of the configuration's status (the -/// configuration named by the root, `+0x80`). -const CONFIG_STATUS_POINTER: usize = 0x3b50; -/// Offset in the power configuration of the GPU address of the AON table. -const AON_POINTER: usize = 0x6318; -/// Offset in the power configuration of a random word the firmware expects. -const ENTROPY: usize = 0x63d4; - -/// Nonzero words of the power configuration and the auxiliary data that follows it, offsets -/// from the start of the power configuration. -const POWER_CONFIG_WORDS: Words = &[ - (0x4, PERIOD_CLOCKS, 2), - (0x10, 0x4, 1), - (0x14, 0x3f800000, 1), - (0x2c, 0x1, 2), - (0x40, 0x64, 1), - (0x44, 0x1, 1), - (0x48, 0x3e8, 1), - (0x50, 0x64, 1), - (0x60, 0x4, 1), - (0x64, 0x3f800000, 1), - (0x6c, 0x1, 2), - (0x80, 0x64, 1), - (0x84, 0x1, 1), - (0x88, 0x3e8, 1), - (0x90, 0x64, 1), - (0xc0, 0x3f828f5c, 11), - (0x9b8, 0x271, 1), - (0x9c4, 0x3f7f2e9f, 1), - (0x9cc, 0x3b516154, 1), - (0x9d4, 0x3ca59586, 1), - (0x9e4, 0x40a90fdb, 1), - (0x9ec, 0xbc6d0cfe, 1), - (0x9f0, 0x64, 1), - (0x9f4, 0x3e8, 2), - (0xa38, 0x3e8, 1), - (0xa70, 0x8, 1), - (0xa7c, 0x3f000000, 1), - (0xa84, 0x3f000000, 1), - (0xa8c, 0x3e3bd7b3, 1), - (0xa98, 0x47800000, 1), - (0xa9c, 0x3ecf2e49, 1), - (0xaa8, 0x64, 1), - (0xaac, 0x3e8, 1), - (0xac8, 0x10, 1), - (0xad0, 0x5dc00, 1), - (0xb28, 0x50, 1), - (0xb30, 0x55, 1), - (0xb34, 0x32, 1), - (0xb38, 0x6, 1), - (0xb40, 0x6, 1), - (0xb44, 0x1, 1), - (0xb4c, 0x3f4ccccd, 1), - (0xb50, 0x3f7eb852, 1), - (0xb54, 0x3e4ccccd, 1), - (0xb58, 0x3ba3d70a, 1), - (0xb5c, 0x3f64c986, 2), - (0xb68, 0x42be0000, 1), - (0xb6c, 0x403ea7f0, 2), - (0xb74, 0x41179436, 1), - (0xb78, 0x64, 1), - (0xb7c, 0x3e8, 1), - (0xb80, 0x64, 1), - (0xb8c, 0x50, 1), - (0xbc0, 0x64, 1), - (0x1bec, 0xff000000, 1), - (0x1bf0, 0xff7fff7f, 255), - (0x1fec, 0x7fff7f, 1), - (0x21f0, 0x47800000, 1), - (0x21f4, 0x42c80000, 1), - (0x21f8, 0x447a0000, 1), - (0x21fc, 0x62a, 1), - (0x2210, 0x64, 1), - (0x2218, 0x42c80000, 1), - (0x221c, 0x3e8, 1), - (0x2224, 0x3f4ccccd, 1), - (0x2228, 0x3e4ccccd, 1), - (0x2268, TEMP_SENSOR_MASK, 1), - (0x2270, 0x1f40, 1), - (0x2278, 0x4, 1), - (0x228c, 0x3f800000, 1), - (0x2294, 0x414eeeee, 1), - (0x22a0, 0x47800000, 1), - (0x22a4, 0x42140000, 1), - (0x22b0, 0x64, 1), - (0x22b4, 0x3e8, 2), - (0x22c4, 0x2af8, 1), - (0x22f8, 0x3e8, 1), - (0x2728, 0xb8520000, 1), - (0x272c, 0x3f7e, 1), - (0x2730, 0xd70a0000, 1), - (0x2734, 0x3ba3, 1), - (0x273c, 0x43c8, 1), - (0x2748, 0x4780, 1), - (0x274c, 0x42c8, 1), - (0x2754, 0x384bac8, 1), - (0x2758, 0x3e80000, 2), - (0x2764, 0x80000000, 1), - (0x2768, 0x26ac453b, 1), - (0x279c, 0x3e80000, 1), - (0x27b8, 0xc80000, 1), - (0x2870, 0x3e8, 1), - (0x28c0, 0x1, 1), - (0x28e0, 0x3f800000, 1), - (0x28e4, 0x4, 1), - (0x28e8, 0x8, 1), - (0x28ec, 0x2ee00, 1), - (0x29bc, 0x7c, 1), - (0x29c8, 0x3f77bdef, 1), - (0x29d0, 0x3d042108, 1), - (0x29d8, 0x3d25bea1, 1), - (0x29e4, 0x47800000, 1), - (0x29e8, 0x3f4f2e49, 1), - (0x29f4, 0x28, 1), - (0x29f8, 0x3e8, 2), - (0x2a14, 0xf8, 1), - (0x2a1c, 0x5ad200, 1), - (0x2a3c, 0x3e8, 1), - (0x2b4c, 0x32, 1), - (0x2b50, 0x1, 1), - (0x2b58, 0x3f638e39, 1), - (0x2b5c, 0x3f2aaaab, 1), - (0x2b60, 0x3de38e39, 1), - (0x2b64, 0x3eaaaaab, 1), - (0x2b68, 0xbecccccd, 2), - (0x2b74, 0x47800000, 1), - (0x2b78, 0xc0a00000, 2), - (0x2b84, 0x64, 1), - (0x2b88, 0x3e8, 1), - (0x2b8c, 0x64, 1), - (0x2b94, 0x45fa0000, 1), - (0x2b98, 0x578, 1), - (0x2ba4, 0x48, 1), - (0x2ba8, 0x18, 1), - (0x2bac, 0x1a5e00, 1), - (0x2bb4, 0x8ca00, 1), - (0x2bc0, 0x47800000, 1), - (0x2be8, 0x1f40, 1), - (0x2bec, 0x4, 1), - (0x2bf0, 0x9c4, 1), - (0x2bf4, 0x258, 1), - (0x2bf8, 0x2, 1), - (0x2bfc, 0x4, 1), - (0x2c00, 0x32, 1), - (0x2c30, 0x47800000, 1), - (0x2c40, 0x64, 1), - (0x2c44, 0x3e8, 1), - (0x2e2c, 0x4, 1), - (0x2e38, 0x3f780000, 1), - (0x2e40, 0x3d000000, 1), - (0x2e48, 0x40a00000, 1), - (0x2e54, 0x47800000, 1), - (0x2e58, 0x41700000, 1), - (0x2e64, 0x384, 1), - (0x2e68, 0x3e8, 2), - (0x2e78, 0x3e8, 1), - (0x2eac, 0x3e8, 1), - (0x2eec, 0x64, 1), - (0x2f08, TEMP_SENSOR_MASK, 1), - (0x2f10, 0x7d, 1), - (0x2f14, 0x1, 1), - (0x3b58, TEMP_SENSOR_MASK, 1), - (0x3b60, 0x1, 1), - (0x5b64, 0x40a00000, 1), - (0x6320, u32::from_be_bytes(*b"g0TR"), 1), - (0x6324, u32::from_be_bytes(*b"g1TR"), 1), - (0x6360, 0x3, 1), - (0x6368, u32::from_be_bytes(*b"r0TR"), 1), - (0x6370, 0x1, 1), - (0x6420, 0x1, 1), -]; - -/// The power configuration and auxiliary data of the 25G83 build, at bundle -/// [`POWER_CONFIG`]. The main configuration names it. -#[repr(C)] -pub(crate) struct PowerConfig25G83(pub(crate) [u8; 0x6510]); - -// SAFETY: `PowerConfig25G83` is a byte array. -unsafe impl Zeroable for PowerConfig25G83 {} - -static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); - -impl PowerConfig25G83 { - /// Initializes the power configuration. - pub(crate) fn init( - &mut self, - config_va: u64, - aon_va: u64, - entropy: u32, - max_power_mw: u32, - leakage: &[u32; 56], - ) { - self.0.fill(0); - fill_words(&mut self.0, POWER_CONFIG_WORDS); - for (index, word) in leakage.iter().enumerate() { - let at = LEAKAGE_OFFSET + 4 * index; - self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); - } - // All copies of the power ceiling must describe the same chip. - for offset in [0xa04, 0xabc, 0x28c8, 0x28cc, 0x28d0, 0x2a08] { - self.0[offset..offset + 4].copy_from_slice(&max_power_mw.to_le_bytes()); - } - let power = crate::float::F32::from(max_power_mw).to_bits(); - for offset in [0x9e0, 0x2a04] { - self.0[offset..offset + 4].copy_from_slice(&power.to_le_bytes()); - } - self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] - .copy_from_slice(&(config_va + 0x80).to_le_bytes()); - self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); - self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); - } -} - -/// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU -/// cluster 1 and SoC sensors. -pub(crate) const AON_TABLE: [u32; 9] = [ - u32::from_be_bytes(*b"g0TR"), - 0x006b_0001, - 0x0000_1401, - u32::from_be_bytes(*b"g1TR"), - 0x00b5_0001, - 0x0000_1401, - u32::from_be_bytes(*b"r0TR"), - 0x00c1_0001, - 0x0000_1401, -]; - -/// Nonzero words of the power-state object. The fast-die limiter reads its target -/// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero -/// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. -pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index ca2b6604a302b..e07fbfb485a10 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -234,7 +234,7 @@ impl McacheRange { return Err(EINVAL); } Ok(Self { - start: start | u64::from(context) << Self::CONTEXT_SHIFT | Self::UNK_START, + start: start | (context as u64) << Self::CONTEXT_SHIFT | Self::UNK_START, end: end | Self::UNK_END | 1 << (Self::CLASS_SHIFT + class as u32), }) } @@ -278,20 +278,6 @@ impl RegisterArray { self.size = (size_of_val(writes) as u32) << 16 | count as u32; Ok(()) } - - /// Stores `writes` without the entry at `skip`, exactly as [`Self::set`] stores the - /// remaining writes, without first copying them elsewhere. - pub(crate) fn set_without(&mut self, self_va: u64, writes: &[RegisterWrite], skip: usize) -> Result { - let count = writes.len().checked_sub(1).ok_or(EINVAL)?; - if skip > count || count > REGISTER_ARRAY_ENTRIES { - return Err(EINVAL); - } - self.entries[..skip].copy_from_slice(&writes[..skip]); - self.entries[skip..count].copy_from_slice(&writes[skip + 1..]); - self.self_va = self_va; - self.size = ((count * size_of::()) as u32) << 16 | count as u32; - Ok(()) - } } /// Binding of one [`RegisterArray`] to a kick entry. @@ -311,7 +297,7 @@ impl RegisterArrayBinding { return Err(EINVAL); } Ok(Self { - word: packed_va(array_va)? | u64::from(checkpoint) << Self::CHECKPOINT_SHIFT, + word: packed_va(array_va)? | (checkpoint as u64) << Self::CHECKPOINT_SHIFT, }) } @@ -346,7 +332,7 @@ impl McacheTable { fn word(self) -> u64 { (self.va >> PACKED_VA_SHIFT) << Self::VA_SHIFT - | u64::from(self.count - 1) << Self::COUNT_SHIFT + | ((self.count - 1) as u64) << Self::COUNT_SHIFT } } @@ -458,8 +444,8 @@ impl KickEntry { } let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT - | u64::from(args.qid) << Self::QID_SHIFT - | u64::from(args.priority) << Self::PRIORITY_SHIFT; + | (args.qid as u64) << Self::QID_SHIFT + | (args.priority as u64) << Self::PRIORITY_SHIFT; if args.scs_enabled { header |= Self::SCS_ENABLED; } @@ -492,9 +478,9 @@ impl KickEntry { mcache, descriptor_va: args.descriptor_va, queue_va: args.queue_va, - dependency_control: u64::from(mask) - | u64::from(args.qos.slot) << Self::QOS_SLOT_SHIFT - | u64::from(args.qos.class) << Self::QOS_CLASS_SHIFT + dependency_control: mask as u64 + | (args.qos.slot as u64) << Self::QOS_SLOT_SHIFT + | (args.qos.class as u64) << Self::QOS_CLASS_SHIFT | Self::UNK_DEPENDENCY_CONTROL, dependencies, event_mask: args.event_mask, diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 1cb46fce2d602..b6bb6aaea860f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -23,9 +23,6 @@ pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; pub(crate) mod qos; -pub(crate) mod variant; - -pub(crate) use variant::Variant; /// Clears a firmware structure in place. /// @@ -33,7 +30,7 @@ pub(crate) use variant::Variant; pub(crate) fn clear(object: &mut T) { // SAFETY: `T: Zeroable` guarantees that all-zero bytes are a valid `T`, // and `object` is a valid, exclusive reference to one `T`. - unsafe { core::ptr::write_bytes(core::ptr::from_mut(object), 0, 1) }; + unsafe { core::ptr::write_bytes(object as *mut T, 0, 1) }; } /// Returns `va + offset`, or `EINVAL` if the sum overflows. diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 66e7d07db48a5..48b110a4193e3 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -20,8 +20,6 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::prelude::*; -use super::variant::PointerLayout; - use super::kick::{KickTimestamp, QID_MAX}; /// Firmware scheduling profile of a queue. @@ -96,8 +94,6 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, - /// [`InitBufferManager`] (25G83 firmware). - InitBufferManager = 5, } /// The engine that executes a queue's work. @@ -119,29 +115,26 @@ pub(crate) enum DataMaster { /// /// - creation: the whole block, [`PointerBlock::new`], before the queue is /// first published; -/// - publication: the producer, with one aligned 32-bit store per publication +/// - publication: `producer`, with one aligned 32-bit store per publication /// step, after a barrier that orders the work pointers and commands before /// it; /// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. -/// -/// Where the producer and the recovery target live depends on the firmware -/// build ([`PointerLayout`]). #[repr(C)] #[derive(Debug)] pub(crate) struct PointerBlock { /// Item-ring position up to which the firmware has finished commands. - /// Free ring space is measured from it, so it bounds the producer. + /// Free ring space is measured from it, so it bounds `producer`. pub(crate) done: AtomicU32, pub(crate) unk_04: [u32; 11], /// Written by the firmware. The host treats the queue as idle only when - /// this cursor, `done` and the producer are equal. + /// this cursor, `done` and `producer` are equal. pub(crate) unk_30: AtomicU32, pub(crate) unk_34: [u32; 3], - /// 26A428: item-ring position after the last work pointer the host published. - word_40: AtomicU32, + /// Item-ring position after the last work pointer the host published. + pub(crate) producer: AtomicU32, pub(crate) unk_44: [u32; 3], - /// 26A428: on recovery, the host copies the producer here before acknowledging resume. - word_50: AtomicU32, + /// On recovery, the host copies producer here before acknowledging resume. + pub(crate) recovery_target: u32, pub(crate) unk_54: [u32; 3], /// Number of entries in the item ring. pub(crate) entries: u32, @@ -150,62 +143,29 @@ pub(crate) struct PointerBlock { static_assert!(size_of::() == 0x80); static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); -static_assert!(core::mem::offset_of!(PointerBlock, word_40) == 0x40); -static_assert!(core::mem::offset_of!(PointerBlock, word_50) == 0x50); +static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); // SAFETY: PointerBlock contains only integers and integer atomics. unsafe impl Zeroable for PointerBlock {} impl PointerBlock { - /// Offsets of the two host words a [`PointerLayout`] may name. - const WORD_40: usize = core::mem::offset_of!(Self, word_40); - const WORD_50: usize = core::mem::offset_of!(Self, word_50); - - /// Whether `offset` is one of the host words a [`PointerLayout`] may name. - const fn is_host_word(offset: usize) -> bool { - offset == Self::WORD_40 || offset == Self::WORD_50 - } - - /// Whether `layout` names host words of the block, and two different ones when it has a - /// recovery target. Every firmware variant's layout is checked at build time. - pub(crate) const fn valid_layout(layout: &PointerLayout) -> bool { - match layout.recovery_target { - None => Self::is_host_word(layout.producer), - Some(target) => { - Self::is_host_word(layout.producer) - && Self::is_host_word(target) - && target != layout.producer - } - } - } + pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); + pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); /// Returns the cursors of an empty item ring. - pub(crate) fn new(layout: &PointerLayout) -> Result { - let block = Self { + pub(crate) const fn new() -> Self { + Self { done: AtomicU32::new(0), unk_04: [0; 11], unk_30: AtomicU32::new(0), unk_34: [0; 3], - word_40: AtomicU32::new(0), + producer: AtomicU32::new(0), unk_44: [0; 3], - word_50: AtomicU32::new(0), + recovery_target: u32::MAX, unk_54: [0; 3], entries: ITEM_RING_ENTRIES, unk_64: [0; 7], - }; - if let Some(target) = layout.recovery_target { - block.word(target)?.store(u32::MAX, Ordering::Relaxed); - } - Ok(block) - } - - /// The host word at `offset`, which must be one a [`PointerLayout`] may name. - fn word(&self, offset: usize) -> Result<&AtomicU32> { - match offset { - Self::WORD_40 => Ok(&self.word_40), - Self::WORD_50 => Ok(&self.word_50), - _ => Err(EINVAL), } } @@ -214,14 +174,12 @@ impl PointerBlock { /// Used when a render queue pair that was kicked before is handed to a new /// owner, with `base` the position the host tracks for the new owner's /// queues. Only valid while the firmware does not run the queue, with all - /// three cursors zero. Writes `done`, `unk_30` and the producer, in this + /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this /// order, with single stores; the caller issues a full barrier afterwards. - pub(crate) fn rebase(&self, layout: &PointerLayout, base: u32) -> Result { - let producer = self.word(layout.producer)?; + pub(crate) fn rebase(&self, base: u32) { self.done.store(base, Ordering::Relaxed); self.unk_30.store(base, Ordering::Relaxed); - producer.store(base, Ordering::Relaxed); - Ok(()) + self.producer.store(base, Ordering::Relaxed); } } @@ -313,12 +271,12 @@ impl QueueRecord { job_list_va: args.job_list_va, unk_18: [0; 0xc], unk_24: u32::MAX, - priority: [u32::from(args.policy.priority()); 2], + priority: [args.policy.priority() as u32; 2], unk_30: [0; 6], unk_36: u16::MAX, unk_38: 0, unk_3c: 0, - policy: u32::from(args.policy.policy()), + policy: args.policy.policy() as u32, unk_44: u32::MAX, owner_pid: args.owner_pid, unk_4c: [0; 0x50], @@ -349,12 +307,12 @@ impl QueueRecord { /// Replaces the priority and policy words of an idle queue's record when they differ /// from `policy`. pub(crate) fn set_policy(&mut self, policy: Policy) { - let priority = [u32::from(policy.priority()); 2]; + let priority = [policy.priority() as u32; 2]; if { self.priority } != priority { self.priority = priority; } - if { self.policy } != u32::from(policy.policy()) { - self.policy = u32::from(policy.policy()); + if { self.policy } != policy.policy() as u32 { + self.policy = policy.policy() as u32; } } } @@ -522,7 +480,7 @@ impl QueueConfig { binding.pb_va, binding.pb_token, binding.pb_slot, - u32::from(binding.fragment_qid), + binding.fragment_qid as u32, ) } QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), @@ -533,14 +491,14 @@ impl QueueConfig { unk_04: 0, kick_ring_va: args.kick_ring_va, kick_ring_fw_va: args.kick_ring_fw_va, - qid: u16::from(args.qid), - install: u16::from(args.install), + qid: args.qid as u16, + install: args.install as u16, unk_1c: 0, - priority: u16::from(args.policy.priority()), + priority: args.policy.priority() as u16, unk_20: 0, data_master: data_master as u16, unk_24: 0, - completion_seed_valid: u32::from(args.completion_seed.is_some()), + completion_seed_valid: args.completion_seed.is_some() as u32, completion_seed: args.completion_seed.unwrap_or(0), context_id: args.context_id, unk_34: 0, @@ -548,13 +506,13 @@ impl QueueConfig { free_list_generation: args.free_list.generation, free_list_slot: args.free_list.slot, scheduler_va: args.scheduler_va, - qos_update: u32::from(args.qos_update), - qos_slot: u16::from(args.qos_slot), + qos_update: args.qos_update as u32, + qos_slot: args.qos_slot as u16, unk_58: 0, owner_pid: args.owner_pid, policy: args.policy.policy(), unk_60: 0, - context_update: u16::from(args.context_update), + context_update: args.context_update as u16, unk_64: 0, unk_66: Self::UNK_66, unk_68: [0; 6], @@ -567,37 +525,6 @@ impl QueueConfig { } } -/// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling -/// descriptor that uses a parameter buffer. The block count is read from the manager. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct InitBufferManager { - /// [`CommandTag::InitBufferManager`]. - pub(crate) tag: u32, - /// UAT context of the pass. - pub(crate) context_id: u32, - /// Buffer slot of the parameter buffer. - pub(crate) pb_slot: u32, - /// GPU address of the parameter-buffer state object. - pub(crate) pb_va: u64, - /// Tiling stamp value of the pass. - pub(crate) stamp_value: u32, -} - -static_assert!(size_of::() == 0x18); - -impl InitBufferManager { - pub(crate) fn new(context_id: u32, pb_slot: u32, pb_va: u64, stamp_value: u32) -> Self { - Self { - tag: CommandTag::InitBufferManager as u32, - context_id, - pb_slot, - pb_va, - stamp_value, - } - } -} - /// Announcement of a newly written kick entry of a queue (tag 14). /// /// A record slot larger than this structure must be zero beyond it. @@ -637,7 +564,7 @@ impl KickAnnounce { } Ok(Self { tag: CommandTag::KickAnnounce as u32, - qid: u16::from(qid), + qid: qid as u16, count: 1, priority, timestamp: timestamp.get(), @@ -676,7 +603,7 @@ impl KickPredecessor { } Ok(Self { tag: CommandTag::KickPredecessor as u32, - qid: u16::from(qid), + qid: qid as u16, previous: previous.get(), unk_0e: Self::UNK_0E, }) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index d0000282cef82..85a2385b421b8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -48,53 +48,10 @@ pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; const USER_BASE: u64 = 0x10_0000_0000; /// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. const VDM_OFFSET_LIMIT: u64 = 1 << 39; -/// The deflake (context) buffer holds three sub-buffers: at its start and at two offsets -/// that follow the control words of each cluster. -#[derive(Copy, Clone, Debug)] -pub(crate) struct Deflake { - /// Offsets of the second and third sub-buffers. - pub(crate) sub: [u64; 2], - /// The host writes the viewport words at `+0x900`; with more clusters that offset lies in - /// the third sub-buffer and the buffer is left to the GPU. - pub(crate) viewport: bool, -} - -impl Deflake { - /// One cluster (A18 Pro). - pub(crate) const ONE_CLUSTER: Deflake = Deflake { - sub: [0x20, 0x2a0], - viewport: true, - }; - /// Two clusters (M4 Pro). - pub(crate) const TWO_CLUSTERS: Deflake = Deflake { - sub: [0xa0, 0x820], - viewport: false, - }; -} - -/// The register programs of a firmware build's render descriptors. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum RegisterPrograms { - /// The A18 Pro firmware's (26A428). - G17P, - /// The 25G83 build's: other tiling and fragment arrays, another tile-state base, and the - /// tiler parameter sub-buffers in the TPC object. - V25G83, -} - -/// The 25G83 tiling and main fragment arrays. -const TA_REGISTERS_25G83: usize = 83; -const FRAGMENT_REGISTERS_25G83: usize = 91; -const TA_CHECKPOINTS_25G83: [u16; 3] = [81, 83, 83]; -const FRAGMENT_CHECKPOINTS_25G83: [u16; 3] = [87, 88, 88]; -/// Index of the free-list slot register (0x14308) in the 25G83 tiling array. -const TA_SLOT_INDEX_25G83: u8 = 79; -/// Index of the free-list slot register (0x14048) in the 25G83 fragment array. -const FRAGMENT_SLOT_INDEX_25G83: u8 = 84; -/// A 25G83 stage with shader-context scratch (ray queries) also writes the SCS enable 0x017d9 = 1, -/// which the compute program and both A18 Pro render programs carry for such work: first in the -/// tiling array, after 0x01739 in the fragment array. Every count and index after it moves by one. -const SCS_ENABLE: u32 = 0x017d9; +/// The deflake buffer holds three sub-buffers: at its start and at these +/// offsets. +const DEFLAKE_20: u64 = 0x20; +const DEFLAKE_2A0: u64 = 0x2a0; const TA_REGISTERS: usize = 73; const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; @@ -147,10 +104,6 @@ const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; /// 32, the utile size bits of the tile-buffer configuration in bits 43:40, /// tiles in X less one from bit 44 and tiles in Y less one from bit 53. const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; -/// The 25G83 build's low tile-state bits, the ones the M4 Pro has run with. With the A18 Pro's, -/// a pass more than 80 tiles wide and with more than 4096 tiles never completes its fragment -/// stage (the 3456x2234 panel is 108 x 70 tiles). -const TILE_STATE_BASE_25G83: u64 = 0x36011 | 1 << 37; const TILE_STATE_LAYERED: u64 = 1 << 32; const TILE_STATE_UTILE_SHIFT: u32 = 28; const TILE_STATE_TILES_X_SHIFT: u32 = 44; @@ -162,14 +115,8 @@ const UTILE_CONFIG_HEIGHT_SHIFT: u32 = 14; const UTILE_CONFIG_SIZE_MASK: u64 = 0xf000; /// The partial-render phases load and store every attached depth and stencil /// surface. -const ZLS_STORE_DEPTH: u64 = 1 << 19; -const ZLS_STORE_STENCIL: u64 = 1 << 18; -const ZLS_LOAD_DEPTH: u64 = 1 << 15; -const ZLS_LOAD_STENCIL: u64 = 1 << 14; -const ZLS_COMPRESS_DEPTH: u64 = 1 << 6; -const ZLS_COMPRESS_STENCIL: u64 = 1 << 8; -/// Bits of the main word a partial render sets for itself. -const ZLS_PARTIAL_CLEAR: u64 = ZLS_COMPRESS_DEPTH | ZLS_COMPRESS_STENCIL | 0xf << 56; +const ZLS_PARTIAL_DEPTH: u64 = 1 << 15 | 1 << 19; +const ZLS_PARTIAL_STENCIL: u64 = 1 << 14 | 1 << 18; const STENCIL_CLEAR_BASE: u32 = 0x300; /// Value of the ZLS swizzle registers and descriptor field. const ZLS_SWIZZLE: u32 = 0x0404_0404; @@ -315,9 +262,6 @@ pub(crate) struct RenderResources { pub(crate) tpc_va: u64, /// Size of the tail-pointer cache the target needs. pub(crate) tpc_size: u64, - /// GPU addresses of the tiler parameter sub-buffers (entry 0 is the TPC); zero for - /// an empty one and on other GPUs. - pub(crate) gtp_va: [u64; 8], /// GPU address of the deflake buffer. pub(crate) deflake_va: u64, /// GPU address of the tiling status word. @@ -370,10 +314,6 @@ pub(crate) struct RenderArgs<'a> { pub(crate) ta: RenderStage, /// Fragment stage. pub(crate) fragment: RenderStage, - /// Register programs of the firmware build. - pub(crate) programs: RegisterPrograms, - /// Deflake-buffer layout of the GPU. - pub(crate) deflake: Deflake, } impl RenderArgs<'_> { @@ -417,12 +357,12 @@ impl Geometry { ) { return Err(EINVAL); } - let width = u64::from(pass.width); - let height = u64::from(pass.height); + let width = pass.width as u64; + let height = pass.height as u64; let tiles_x = width.div_ceil(TILE_SIZE); let tiles_y = height.div_ceil(TILE_SIZE); - let utiles_x = TILE_SIZE / u64::from(pass.utile_width); - let utiles_y = TILE_SIZE / u64::from(pass.utile_height); + let utiles_x = TILE_SIZE / pass.utile_width as u64; + let utiles_y = TILE_SIZE / pass.utile_height as u64; let utiles = utiles_x * utiles_y; let mtile_x = tiles_x.div_ceil(MACRO_TILES).next_multiple_of(4); let mtile_y = tiles_y.div_ceil(MACRO_TILES).next_multiple_of(4); @@ -449,7 +389,6 @@ struct PassWords { layer_mode: u64, tib_blocks: u64, partial_zls_control: u64, - partial_zls_reload: u64, aux_fb_flags: u64, bg_rsrc_spec: u64, bg_va: u64, @@ -472,7 +411,7 @@ impl PassWords { // The tile configuration carries the empty-tile request for every // target; the tiling layer mode only for layered ones. let mut tile_config = TILE_CONFIG_BASE; - let mut layer_mode = LAYER_MODE_BASE | (u64::from(pass.layers) - 1); + let mut layer_mode = LAYER_MODE_BASE | (pass.layers as u64 - 1); if layered { tile_config |= TILE_CONFIG_LAYERED; layer_mode |= LAYER_MODE_LAYERED; @@ -485,27 +424,12 @@ impl PassWords { tile_config |= TILE_CONFIG_PROCESS_EMPTY_TILES; } - // The partial store keeps the main word, loads included, and stores and compresses each - // plane it has. The reload arrays add the loads, which a clearing pass doesn't have. - let mut partial_zls_control = pass.zls_control & !ZLS_PARTIAL_CLEAR; - if pass.depth.va != 0 { - partial_zls_control |= ZLS_STORE_DEPTH; - } - if pass.depth.compression_va != 0 { - partial_zls_control |= ZLS_COMPRESS_DEPTH; - } - if pass.stencil.va != 0 { - partial_zls_control |= ZLS_STORE_STENCIL; - } - if pass.stencil.compression_va != 0 { - partial_zls_control |= ZLS_COMPRESS_STENCIL; - } - let mut partial_zls_reload = partial_zls_control; + let mut partial_zls_control = pass.zls_control; if pass.depth.va != 0 { - partial_zls_reload |= ZLS_LOAD_DEPTH; + partial_zls_control |= ZLS_PARTIAL_DEPTH; } if pass.stencil.va != 0 { - partial_zls_reload |= ZLS_LOAD_STENCIL; + partial_zls_control |= ZLS_PARTIAL_STENCIL; } let mut aux_fb_flags = AUX_FB_FLAGS_BASE; @@ -517,23 +441,22 @@ impl PassWords { } let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { true => spec, - false => spec & u64::from(u32::MAX), + false => spec & u32::MAX as u64, }; - let usc = |offset: u32| offset_va(pass.usc_base, u64::from(offset & !USC_OFFSET_TAG_MASK)); + let usc = |offset: u32| offset_va(pass.usc_base, (offset & !USC_OFFSET_TAG_MASK) as u64); Ok(Self { - utile_config: (u64::from(pass.utile_width) / 16) << UTILE_CONFIG_WIDTH_SHIFT - | (u64::from(pass.utile_height) / 16) << UTILE_CONFIG_HEIGHT_SHIFT - | u64::from(pass.samples_log2), + utile_config: (pass.utile_width as u64 / 16) << UTILE_CONFIG_WIDTH_SHIFT + | (pass.utile_height as u64 / 16) << UTILE_CONFIG_HEIGHT_SHIFT + | pass.samples_log2 as u64, tile_config, layer_mode, - tib_blocks: (u64::from(pass.sample_size) - * u64::from(pass.utile_width) - * u64::from(pass.utile_height) + tib_blocks: (pass.sample_size as u64 + * pass.utile_width as u64 + * pass.utile_height as u64 * samples) .div_ceil(TIB_BLOCK_SIZE), partial_zls_control, - partial_zls_reload, aux_fb_flags, bg_rsrc_spec: rsrc_spec(pass.bg.rsrc_spec), bg_va: usc(pass.bg.offset)?, @@ -568,10 +491,9 @@ fn ta_registers( let layermeta = user_offset(r.layermeta_va)?; let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; let tpc = user_offset(r.tpc_va)?; - let [deflake_20, deflake_2a0] = args.deflake.sub; - let deflake = u64::from(user_offset32(r.deflake_va)?); - let deflake_20 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_20)?)?); - let deflake_2a0 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_2a0)?)?); + let deflake = user_offset32(r.deflake_va)? as u64; + let deflake_20 = user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)? as u64; + let deflake_2a0 = user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)? as u64; let vdm = user_offset(p.vdm_va)?; if vdm >= VDM_OFFSET_LIMIT { return Err(EINVAL); @@ -639,7 +561,7 @@ fn ta_registers( RegisterWrite::new(0x0d471, rt.scratch.tail_word()), RegisterWrite::new(0x1a0f1, rt.scratch.stack_word()), RegisterWrite::new(0x10799, rt.scratch.window_word()), - RegisterWrite::new(0x1c830, u64::from(r.pb_slot)), + RegisterWrite::new(0x1c830, r.pb_slot as u64), RegisterWrite::new(0x1ca30, r.pm_scratch), RegisterWrite::new(0x16c39, r.pm_scratch), RegisterWrite::new(0x1c910, r.pm_metric), @@ -651,7 +573,7 @@ fn ta_registers( RegisterWrite::new(0x10209, key), RegisterWrite::new(0x1c9f0, key), RegisterWrite::new(0x14320, key), - RegisterWrite::new(0x14308, u64::from(r.free_list_slot)), + RegisterWrite::new(0x14308, r.free_list_slot as u64), RegisterWrite::new(0x14318, r.ta_status_va | 1), RegisterWrite::new(0x01740, 1), RegisterWrite::new(0x1c880, deflake_2a0), @@ -659,136 +581,6 @@ fn ta_registers( ]) } -/// Tiler segment control of a two-cluster pass, what macOS writes. -const VDM_SEGMENT_CONTROL: u64 = 0x20_0440; -const VDM_SEGMENT_INDEX: u64 = 4; -/// Pages the tiler keeps out of a partial render's segments: without them the pages a pass -/// returns shrink to none and the tiler can't advance. Fifteen blocks, macOS's minimum heap. -const MIN_FREE_TVB_PAGES: u64 = 0x3c; - -/// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage -/// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the -/// M4 Pro has run with. Returns the array and its used entry count. -fn ta_registers_25g83( - args: &RenderArgs<'_>, - g: &Geometry, - w: &PassWords, -) -> Result<([RegisterWrite; TA_REGISTERS_25G83 + 1], usize)> { - let a = ta_registers(args, g, w)?; - let r = args.resources; - let gtp = |index: usize| -> Result { - match r.gtp_va[index] { - 0 => Ok(0), - va => user_offset(va), - } - }; - let find = |reg: u32, nth: usize| -> Result { - a.iter() - .filter(|write| write.reg == reg) - .nth(nth) - .copied() - .ok_or(EINVAL) - }; - let gtp1 = gtp(1)?; - let gtp2 = gtp(2)?; - let gtp6 = gtp(6)? & !127; - let scs = usize::from(args.ray_query[0].scratch.enabled()); - let mut out = [RegisterWrite::new(0, 0); TA_REGISTERS_25G83 + 1]; - let mut n = 0; - let mut push = |write: RegisterWrite| { - out[n] = write; - n += 1; - }; - if scs != 0 { - push(find(SCS_ENABLE, 0)?); - } - for reg in [0x10141, 0x1c039, 0x1c9c8] { - push(find(reg, 0)?); - } - push(RegisterWrite::new(0x1c041, gtp1)); - push(RegisterWrite::new(0x1c9d0, gtp1)); - for reg in [ - 0x1c0a1, 0x1c031, 0x1c9c0, 0x1c051, 0x1c061, 0x10149, 0x10139, 0x10111, 0x1c9b0, 0x10119, - 0x1c9b8, 0x1c958, 0x1c950, 0x1c930, - ] { - push(find(reg, 0)?); - } - push(find(0x1c880, 0)?); - push(RegisterWrite::new(0x1c948, gtp(4)?)); - push(RegisterWrite::new(0x1c888, gtp(5)?)); - push(RegisterWrite::new(0x1c890, VDM_SEGMENT_CONTROL)); - push(RegisterWrite::new(0x1c918, VDM_SEGMENT_INDEX)); - push(find(0x1c079, 0)?); - push(find(0x1c9d8, 0)?); - push(RegisterWrite::new(0x1c089, gtp2)); - push(RegisterWrite::new(0x1c9e0, gtp2)); - push(RegisterWrite::new(0x16c41, gtp6)); - push(RegisterWrite::new(0x1ca40, gtp6)); - push(RegisterWrite::new(0x1c9a8, MIN_FREE_TVB_PAGES)); - push(RegisterWrite::new(0x1c920, gtp(7)?)); - for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { - push(find(reg, 0)?); - } - push(RegisterWrite::new(0x10131, 0x88)); - for reg in [ - 0x10121, 0x10129, 0x101b9, 0x1c069, 0x1c071, 0x1c081, 0x1c0a9, 0x10171, 0x10169, 0x0a309, - 0x1c8e0, 0x1c8e8, - ] { - push(find(reg, 0)?); - } - push(find(0x1c898, 0)?); - for reg in [ - 0x101e1, 0x1c9e8, 0x1a099, 0x1a0a1, 0x1a069, 0x1a071, 0x1a0c9, 0x1a0d1, 0x101c9, 0x0d471, - 0x1a0f1, 0x10799, 0x1c830, 0x1ca30, 0x16c39, 0x1c910, 0x0a5a1, 0x0d419, 0x1ca10, 0x014a1, - 0x0a349, 0x10209, 0x1c9f0, 0x14320, 0x14308, 0x14318, - ] { - push(find(reg, 0)?); - } - push(find(0x1c880, 1)?); - push(find(0x1c898, 1)?); - if n != TA_REGISTERS_25G83 + scs || out[TA_SLOT_INDEX_25G83 as usize + scs].reg != 0x14308 { - return Err(EINVAL); - } - Ok((out, n)) -} - -/// The 25G83 main fragment array: the A18 Pro's (without 0x17d9 unless the stage has SCS), with -/// 0x160b8 = 0 after the tile state and 0x160a8 = 0x20 before the closing 0x16068, as the M4 -/// Pro has run with. Returns the array and its used entry count. -fn fragment_registers_25g83( - args: &RenderArgs<'_>, - g: &Geometry, - w: &PassWords, -) -> Result<([RegisterWrite; FRAGMENT_REGISTERS_25G83 + 1], usize)> { - let a = fragment_registers(args, g, w); - let scs = usize::from(args.ray_query[1].scratch.enabled()); - let mut out = [RegisterWrite::new(0, 0); FRAGMENT_REGISTERS_25G83 + 1]; - let mut n = 0; - let mut tile_state = false; - for (index, write) in a.iter().enumerate() { - if write.reg == SCS_ENABLE && scs == 0 { - continue; - } - if index == a.len() - 1 { - *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160a8, 0x20); - n += 1; - } - *out.get_mut(n).ok_or(EINVAL)? = *write; - n += 1; - if write.reg == 0x16068 && !tile_state { - tile_state = true; - *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160b8, 0); - n += 1; - } - } - if n != FRAGMENT_REGISTERS_25G83 + scs - || out[FRAGMENT_SLOT_INDEX_25G83 as usize + scs].reg != 0x14048 - { - return Err(EINVAL); - } - Ok((out, n)) -} - fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, @@ -796,12 +588,7 @@ fn fragment_registers( ) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; - let tile_state_base = if args.programs == RegisterPrograms::V25G83 { - TILE_STATE_BASE_25G83 - } else { - TILE_STATE_BASE - }; - let mut tile_state = tile_state_base + let mut tile_state = TILE_STATE_BASE | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; @@ -822,8 +609,8 @@ fn fragment_registers( RegisterWrite::new(0x15381, w.eot_va), RegisterWrite::new(0x15369, w.bg_rsrc_spec), RegisterWrite::new(0x15371, w.bg_va), - RegisterWrite::new(0x15131, u64::from(p.merge_upper_x)), - RegisterWrite::new(0x15139, u64::from(p.merge_upper_y)), + RegisterWrite::new(0x15131, p.merge_upper_x as u64), + RegisterWrite::new(0x15139, p.merge_upper_y as u64), RegisterWrite::new(0x100a1, 0), RegisterWrite::new(0x15069, 0), RegisterWrite::new(0x15071, 0), @@ -838,15 +625,15 @@ fn fragment_registers( RegisterWrite::new(0x15109, p.scissor_va), RegisterWrite::new(0x15101, p.depth_bias_va), RegisterWrite::new(0x15021, w.aux_fb_flags), - RegisterWrite::new(0x15211, u64::from(p.height) << 32 | u64::from(p.width)), + RegisterWrite::new(0x15211, (p.height as u64) << 32 | p.width as u64), RegisterWrite::new(0x15049, UNK_15049), RegisterWrite::new(0x10051, w.tib_blocks), RegisterWrite::new(0x15321, p.depth_dimensions), - RegisterWrite::new(0x15301, u64::from(p.depth_clear)), - RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), + RegisterWrite::new(0x15301, p.depth_clear as u64), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), RegisterWrite::new(0x15311, p.occlusion_query_va), RegisterWrite::new(0x15319, p.zls_control), - RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), RegisterWrite::new(0x15351, 0), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x15331, p.depth.va), @@ -895,11 +682,11 @@ fn fragment_registers( RegisterWrite::new(0x160e0, ids), RegisterWrite::new(0x01499, ids), RegisterWrite::new(0x0a341, ids), - RegisterWrite::new(0x1c838, u64::from(r.pb_slot)), + RegisterWrite::new(0x1c838, r.pb_slot as u64), RegisterWrite::new(0x1ca28, r.pm_scratch), RegisterWrite::new(0x10211, key), RegisterWrite::new(0x10420, key), - RegisterWrite::new(0x14048, u64::from(r.free_list_slot)), + RegisterWrite::new(0x14048, r.free_list_slot as u64), RegisterWrite::new(0x14080, r.fragment_status_va | 1), RegisterWrite::new(0x01731, 1), RegisterWrite::new(0x16020, 1), @@ -929,7 +716,7 @@ fn partial_store_registers( RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), ] } @@ -955,13 +742,13 @@ fn partial_resume_registers( RegisterWrite::new(0x15439, p.stencil.compression_stride), RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), - RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_reload), - RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), ] } @@ -972,16 +759,14 @@ fn partial_load_registers( [ RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), - // Only the main end-of-tile program resolves, and it has to reach tiles that only - // earlier segments drew to. Without this their resolve target is never written. - RegisterWrite::new(0x10039, w.tile_config | TILE_CONFIG_PROCESS_EMPTY_TILES), - RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), + RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_reload), - RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), ] } @@ -1055,11 +840,7 @@ pub(crate) struct TaDescriptor { /// QID of the tiling queue. pub(crate) qid: u32, pub(crate) unk_8be: u32, - /// Non-zero on the first kick of a queue pair, like the fragment descriptor's word at the - /// same position. Set on every kick, the firmware also notifies the host of each pass's - /// tiling completion, which no retirement needs. The 25G83 firmware keeps it set on every - /// kick until this is measured there. - pub(crate) first_kick: u32, + pub(crate) unk_8c2: u32, /// Completion value of the tiling kick (see /// [`KickTimestamp::completion_value`]). pub(crate) completion: u64, @@ -1108,6 +889,8 @@ unsafe impl Zeroable for TaDescriptor {} impl TaDescriptor { const ARRAY_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; const UNK_876: u32 = u32::MAX; + const UNK_892: u32 = 1; + const UNK_8C2: u32 = 1; const UNK_932: u8 = 0x44; const UNK_93C: u8 = 1; const UNK_93E_FIRST_KICK: [u8; 2] = [0xd0, 0x91]; @@ -1119,8 +902,7 @@ impl TaDescriptor { let geometry = Geometry::new(pass)?; let words = PassWords::new(pass)?; let registers = ta_registers(args, &geometry, &words)?; - let deflake_2a0_offset = - user_offset32(offset_va(resources.deflake_va, args.deflake.sub[1])?)?; + let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; let sampler_max = sampler_max(pass.sampler_count)?; let stamp_value = args.stamp_value(); @@ -1137,14 +919,12 @@ impl TaDescriptor { let scs_register = args.ray_query[0].scratch.enabled(); self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - if args.programs == RegisterPrograms::V25G83 { - self.checkpoints = TA_CHECKPOINTS_25G83.map(|count| count + u16::from(scs_register)); - let (registers, count) = ta_registers_25g83(args, &geometry, &words)?; - self.registers.set(array_va, ®isters[..count])?; - } else if scs_register { + if scs_register { self.registers.set(array_va, ®isters)?; } else { - self.registers.set_without(array_va, ®isters, TA_SCS_ENABLE_INDEX)?; + let mut trimmed = [registers[0]; TA_REGISTERS]; + trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); + self.registers.set(array_va, &trimmed)?; } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; @@ -1156,24 +936,16 @@ impl TaDescriptor { self.sampler_heap_va = pass.sampler_heap_va; self.sampler_count = pass.sampler_count; self.sampler_max = sampler_max; - // Asks the firmware to grow the heap. macOS keeps that for memoryless passes, which can't - // partial-render, and Honeykrisp has none. 26A428 keeps it: untested there. - self.unk_892 = u32::from(args.programs != RegisterPrograms::V25G83); + self.unk_892 = Self::UNK_892; self.stamp_va = args.ta.stamp_va; self.aux_stamp_va = args.ta.aux_stamp_va; self.stamp_value_2 = stamp_value; self.qid = args.ta.qid.into(); - self.first_kick = match args.programs == RegisterPrograms::V25G83 { - true => 1, - false => u32::from(args.ordinal == 0), - }; + self.unk_8c2 = Self::UNK_8C2; self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; - self.unk_932 = match args.programs == RegisterPrograms::V25G83 { - true => TA_SLOT_INDEX_25G83 + u8::from(scs_register), - false => Self::UNK_932, - }; + self.unk_932 = Self::UNK_932; self.free_list_control_va = resources.free_list_control_va; self.unk_93c = Self::UNK_93C; if args.ordinal == 0 { @@ -1197,7 +969,7 @@ impl TaDescriptor { pass_seq: u8, fragment_busy: bool, ) { - self.fragment_busy = u32::from(fragment_busy); + self.fragment_busy = fragment_busy as u32; self.fragment_kick = fragment.get(); self.pass_seq = [pass_seq; 2]; self.completion = ta.completion_value(pass_seq); @@ -1208,14 +980,11 @@ impl TaDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, - programs: RegisterPrograms, ) -> Result<[Option; 4]> { - let checkpoint = match programs == RegisterPrograms::V25G83 { - true => TA_CHECKPOINTS_25G83[0] as u8, - false => TA_CHECKPOINTS[0] as u8, - } + u8::from(scs_register); - let array = - RegisterArrayBinding::new(offset_va(descriptor_va, Self::ARRAY_OFFSET)?, checkpoint)?; + let array = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::ARRAY_OFFSET)?, + TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), + )?; Ok([Some(array), None, None, None]) } } @@ -1402,18 +1171,13 @@ impl FragmentDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, - programs: RegisterPrograms, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; - let checkpoint = match programs == RegisterPrograms::V25G83 { - true => FRAGMENT_CHECKPOINTS_25G83[0] as usize, - false => FRAGMENT_CHECKPOINTS[0] as usize, - } + usize::from(scs_register); Ok([ - array(0, checkpoint)?, + array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1485,15 +1249,13 @@ impl FragmentSlot { desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); desc.entries = FRAGMENT_REGISTERS as u16 + extra; let registers = fragment_registers(args, &geometry, &words); - if args.programs == RegisterPrograms::V25G83 { - desc.checkpoints = FRAGMENT_CHECKPOINTS_25G83.map(|count| count + extra); - desc.entries = FRAGMENT_REGISTERS_25G83 as u16 + extra; - let (registers, count) = fragment_registers_25g83(args, &geometry, &words)?; - desc.arrays[0].set(arrays[0], ®isters[..count])?; - } else if scs_register { + if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { - desc.arrays[0].set_without(arrays[0], ®isters, FRAGMENT_SCS_ENABLE_INDEX)?; + let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; + trimmed[FRAGMENT_SCS_ENABLE_INDEX..] + .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); + desc.arrays[0].set(arrays[0], &trimmed)?; } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; @@ -1527,15 +1289,12 @@ impl FragmentSlot { desc.aux_stamp_va = args.fragment.aux_stamp_va; desc.stamp_value = args.stamp_value(); desc.qid = args.fragment.qid.into(); - desc.first_kick = u32::from(args.ordinal == 0); + desc.first_kick = (args.ordinal == 0) as u32; desc.current_id = resources.fragment_ids.current.into(); desc.ordinal = args.ordinal as u32; desc.timestamp_start_va = resources.timestamp_va; desc.timestamp_end_va = timestamp_end_va; - desc.unk_21cc = match args.programs == RegisterPrograms::V25G83 { - true => FRAGMENT_SLOT_INDEX_25G83 + u8::from(args.ray_query[1].scratch.enabled()), - false => Self::UNK_21CC, - }; + desc.unk_21cc = Self::UNK_21CC; desc.free_list_control_va = resources.free_list_control_va; desc.unk_21d6 = Self::UNK_21D6; desc.status_va = resources.fragment_status_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs deleted file mode 100644 index 03d5bfccafeca..0000000000000 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! G17 firmware builds. -//! -//! Firmware builds that share the interface version ([`super::initdata::INTERFACE_VERSION`]) -//! still differ in a few shared layouts and queue rules. A [`Variant`] names those differences -//! for one build; each SoC's [`crate::g17::Config`] points at the variant of the firmware it -//! runs. - -/// Where a firmware build keeps the host words of a queue's pointer block -/// ([`super::queue::PointerBlock`]). -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct PointerLayout { - /// Offset of the item-ring producer. - pub(crate) producer: usize, - /// Offset of the word the host copies the producer to before it acknowledges a recovery, - /// if the build has one. - pub(crate) recovery_target: Option, -} - -/// How a firmware build acknowledges the initialization root the host hands it. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum InitAck { - /// With an `INITDATA_ACK` message on the firmware endpoint. - Message, - /// By setting word `+0x14` of the primary status A to 1; it sends no message. - Status, -} - -/// The initialization graph of a firmware build. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum InitGraph { - /// Two instances, as the A18 Pro firmware (26A428) runs. - G17P, - /// One instance carrying both roles, with the 25G83 layouts ([`super::initdata::v25g83`]). - V25G83, -} - -/// The layouts and rules of one G17 firmware build. -pub(crate) struct Variant { - /// The initialization graph. - pub(crate) graph: InitGraph, - /// How the firmware acknowledges its initialization. - pub(crate) init_ack: InitAck, - /// The host ends the power-on transition with `POWER_ON_DONE` once every coprocessor - /// is initialized. - pub(crate) power_on_done: bool, - /// First QID the host may use; the firmware keeps the lower ones for itself. - pub(crate) qid_floor: usize, - /// Host words of a queue's pointer block. - pub(crate) pointers: PointerLayout, - /// The host registers each render pair through the GPU's MMIO queue ports. - pub(crate) mmio_pair_registration: bool, - /// Render kick entries may name an MCache table. - pub(crate) mcache: bool, - /// Renders may set the few-primitives auxiliary framebuffer flag. - pub(crate) few_primitives: bool, - /// The firmware runs ray-traversal work (ray queries and intersection programs). - pub(crate) ray_traversal: bool, - /// Tiling descriptors report whether the fragment queue still had earlier kicks - /// outstanding (`fragment_busy`). - pub(crate) fragment_busy: bool, - /// Render item rings carry lead commands: InitBufferManager before the first tiling pass - /// of a buffer manager. - pub(crate) lead_commands: bool, - /// Register programs of the render descriptors. - pub(crate) programs: super::render::RegisterPrograms, -} - -/// A firmware build a SoC supports, identified by its loaded TEXT. -pub(crate) struct Build { - /// macOS version (major, minor, patch), used to order image candidates. - pub(crate) os_version: [u32; 3], - /// Name, for the log. - pub(crate) name: &'static str, - /// The admitted loaded images, measured from the `fw-text` region. - pub(crate) images: &'static [crate::fwid::FirmwareBuild], - /// Layouts and rules of the build. - pub(crate) variant: &'static Variant, -} - -/// How a SoC's firmware build is determined. -pub(crate) enum Firmware { - /// The SoC runs a single build, which is not identified (A18 Pro). - Fixed(&'static Variant), - /// One of these exact loaded images. The loader's `apple,firmware-version` is only - /// a candidate-ordering hint because multiple OS releases can share an iBoot string. - Identified(&'static [Build]), -} - -/// The A18 Pro firmware (26A428). -pub(crate) const V26A428: Variant = Variant { - graph: InitGraph::G17P, - init_ack: InitAck::Message, - power_on_done: true, - qid_floor: 0, - pointers: PointerLayout { - producer: 0x40, - recovery_target: Some(0x50), - }, - mmio_pair_registration: true, - mcache: true, - few_primitives: true, - ray_traversal: true, - fragment_busy: true, - lead_commands: false, - programs: super::render::RegisterPrograms::G17P, -}; - -/// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). -pub(crate) const V25G83: Variant = Variant { - graph: InitGraph::V25G83, - init_ack: InitAck::Status, - // Never sent on the M4 Pro: the firmware has retired its opening record when it accepts - // its initialization. - power_on_done: false, - // QIDs below 32 belong to the firmware's bootstrap protocol. - qid_floor: 32, - pointers: PointerLayout { - producer: 0x50, - recovery_target: None, - }, - // Whether this build tolerates the MMIO registration is unknown; the M4 Pro has only run - // without it. - mmio_pair_registration: false, - mcache: false, - // A fragment stage with the few-primitives flag hangs. - few_primitives: false, - ray_traversal: true, - // When a tiling pass runs out of memory with this set, the firmware resumes it without - // adding memory and the tiler stops silently; macOS always writes 0. - fragment_busy: false, - lead_commands: true, - programs: super::render::RegisterPrograms::V25G83, -}; - -const _: () = { - assert!(super::queue::PointerBlock::valid_layout(&V26A428.pointers)); - assert!(super::queue::PointerBlock::valid_layout(&V25G83.pointers)); -}; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 578bb7533f3c9..d0dcc85c7f09e 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -16,8 +16,6 @@ use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; -mod v25g83; - const EVENT_RING_RECHECK: u32 = 256; const INDEX_SIZE: usize = 0x10000; @@ -38,30 +36,6 @@ pub(crate) struct ComputeCompletion<'a> { offset: usize, } -/// Records in the compute completion ring that the host scans. -pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { - let slots = abi::COMPLETION_RING_SIZE / 0x40; - if slots < 0x800 { - slots - } else { - 0x800 - } -}; - -/// Timestamp bits of a completion record's start and end words. -const COMPLETION_TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; - -/// What one completion-ring slot gave a scan. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum CompletionFold { - /// No new observation of a pending request; the scan continues. - Passed, - /// Every request now has an observation. - Done, - /// A pending request's record whose payload the firmware is still writing. - Incomplete, -} - impl ComputeCompletion<'_> { pub(crate) fn read(&self, word: usize) -> Result { if word >= 8 { @@ -147,12 +121,20 @@ impl WorkPublication<'_> { // fully initialized padding-free ABI value, and the device mutex owns publication. unsafe { core::ptr::copy_nonoverlapping( - core::ptr::from_ref(self.record).cast::(), + self.record as *const _ as *const u8, self.destination, core::mem::size_of_val(self.record), ); } } + + fn commit(self) -> u32 { + self.copy_record(); + fence(Ordering::SeqCst); + self.producer.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + self.next + } } /// Session-long storage and mappings. Aliases are released before their backing objects. @@ -173,15 +155,6 @@ pub(super) struct InitData { boot_control: usize, boot_state: usize, metrics_fw_va: u64, - /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). - pm_metrics_va: u64, - /// `Config::recovery_slot_keys`. - recovery_slot_keys: Option, - /// Primary status A and status B in the cluster. - status_a: usize, - status_b: usize, - /// Separately allocated views; `None` when they are packed into the bundle. - views: Option<[usize; 5]>, compute_shared: Option, compute_scheduler: Option, compute_active: core::cell::Cell, @@ -201,19 +174,11 @@ struct BuildEnv<'a> { render: &'a mmu::Vm, base: u64, dynamic: Range, - /// GPU clusters, as the GPU reports them. - clusters: u32, } impl BuildEnv<'_> { /// An object at a fixed offset of the kernel window. - fn fixed( - &self, - offset: u64, - size: usize, - prot: mmu::Prot, - cpu: CpuMap, - ) -> Result { + fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { KernelObject::new( self.dev, self.vm, @@ -225,13 +190,7 @@ impl BuildEnv<'_> { } /// An object placed in the dynamic part of the kernel window. - fn dynamic( - &self, - size: usize, - align: u64, - prot: mmu::Prot, - cpu: CpuMap, - ) -> Result { + fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { KernelObject::new( self.dev, self.vm, @@ -276,11 +235,6 @@ impl InitData { boot_control: 0, boot_state: 0, metrics_fw_va: 0, - pm_metrics_va: 0, - recovery_slot_keys: None, - status_a: abi::private::PRIMARY_STATUS_A, - status_b: abi::private::STATUS_B, - views: None, compute_shared: None, compute_scheduler: None, compute_active: core::cell::Cell::new(false), @@ -293,10 +247,6 @@ impl InitData { } } - pub(super) fn pm_metrics_va(&self) -> u64 { - self.pm_metrics_va - } - pub(super) fn metrics_fw_va(&self) -> Result { if self.metrics_fw_va == 0 { return Err(EIO); @@ -336,12 +286,7 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. /// Called for a fresh row or after its free-list release was consumed. - pub(super) fn initialize_compute_pool_blocks( - &self, - slot: u16, - page_list_va: u64, - blocks: usize, - ) -> Result { + pub(super) fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } @@ -418,7 +363,7 @@ impl InitData { .initialize::<[u64; low::RENDER_RUN_LIST_SIZE / 8]>(0, |runs| { for (index, run) in runs.iter_mut().take(COMPUTE_BLOCKS_POPULATED).enumerate() { *run = (low::compute_block(index) & COMPUTE_RUN_ADDRESS_MASK) - | (u64::from(abi::FREE_LIST_BLOCK_PAGES) << 52); + | ((abi::FREE_LIST_BLOCK_PAGES as u64) << 52); } Ok(()) })?; @@ -427,12 +372,6 @@ impl InitData { Ok(()) } - pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { - self.compute_shared - .as_ref() - .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) - } - pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { let shared = self.compute_shared.as_ref().ok_or(EIO)?; if vm.compute_pool_mappings_match(shared.id) { @@ -549,7 +488,7 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - // Compute publications reserve their slots through the port. + 2 => DataMaster::Compute, _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; @@ -563,7 +502,7 @@ impl InitData { core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( - self.state_offset(Role::Primary) + Self::state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -583,7 +522,7 @@ impl InitData { ) -> Result<[u32; 3]> { use super::fw::channels; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); + let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; for (value, offset) in values.iter_mut().zip([ @@ -602,6 +541,16 @@ impl InitData { Ok(values) } + /// Inner queue records must be visible before this outer publication. The returned + /// producer is the retained consumer witness for this slot. + pub(super) fn publish_work( + &self, + priority: u8, + record: &super::fw::channels::WorkSlot, + ) -> Result { + Ok(self.work_publication(priority, record)?.commit()) + } + /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -652,7 +601,7 @@ impl InitData { }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( - self.state_offset(Role::Primary) + Self::state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -700,117 +649,21 @@ impl InitData { Ok(()) } - /// Raw views of the memory publishers share without the device mutex: every work ring, - /// the completion stamps, the QoS table and the recovery state word. They stay valid for - /// this session. - pub(super) fn port_views(&self, mailbox: *const super::Mailbox) -> Result { - use super::fw::{ - channels, - queue::{DataMaster, Policy}, - }; - let ring_of = |priority: u8, engine: DataMaster| -> Result { - let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); - let state = self.object(self.cluster)?; - let word = - |offset: usize| -> Result<*const AtomicU32> { Ok(state.word(base + offset)?) }; - Ok(super::port::RingView { - slots: self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel), - channels::WORK_RING_SLOTS as usize * core::mem::size_of::(), - )?, - consumers: [ - word(abi::ChannelState::CONSUMER)?, - word(abi::ChannelState::UNK_10)?, - ], - producer: word(abi::ChannelState::PRODUCER)?, - }) - }; - let rings = - |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { - let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; - for (priority, view) in views.iter_mut().enumerate().skip(1) { - *view = ring_of(priority as u8, engine)?; - } - Ok(views) - }; - Ok(super::port::Views { - compute: rings(DataMaster::Compute)?, - tiling: rings(DataMaster::Tiling)?, - fragment: rings(DataMaster::Fragment)?, - stamps: self.object(self.stamps[0])?.word(0)?, - qos: self - .qos()? - .pointer(0, core::mem::size_of::())?, - recovery: self - .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, - mailbox, - }) - } - /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. - /// Folds the compute completion records the firmware appended since `start`, in ring - /// order. `seen` holds every slot's control word as of the previous pass over it: the - /// first slot still holding it is where the firmware writes next, and the pass ends there - /// instead of reading the rest of the ring. A pass also ends once `record` has found every - /// request, or at a requested record whose payload is still being written, which the next - /// pass reads again. Returns the slot the next pass starts at. - pub(super) fn fold_compute_completions( + pub(super) fn scan_compute_completions( &self, - start: usize, - seen: &mut [u64], - mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { - let object = self.object(self.completion_rings[0])?; - if seen.len() != COMPUTE_COMPLETION_SLOTS { - return Err(EINVAL); - } - fence(Ordering::Acquire); - let completion = |slot: usize| ComputeCompletion { - object, - offset: slot * 0x40, - }; - super::queue::compute::retirement::walk_completions( - start, - seen, - |slot| completion(slot).read(0), - |slot| record(completion(slot)), - ) - } - - /// Folds every slot of the compute completion ring and records its control word in - /// `seen`. For a pass that stopped before a record the firmware must already have - /// written. Returns the slot after the newest record (latest end timestamp), where the - /// firmware writes next. - pub(super) fn resync_compute_completions( - &self, - seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { + ) -> Result { let object = self.object(self.completion_rings[0])?; - if seen.len() != COMPUTE_COMPLETION_SLOTS { - return Err(EINVAL); - } fence(Ordering::Acquire); - let mut newest = None; - for (slot, seen_control) in seen.iter_mut().enumerate() { - let completion = ComputeCompletion { + for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { + record(ComputeCompletion { object, offset: slot * 0x40, - }; - let control = completion.read(0)?; - *seen_control = control; - if control != 0 { - let end = completion.read(5)? & COMPLETION_TIME_MASK; - if newest.is_none_or(|(_, latest)| end > latest) { - newest = Some((slot, end)); - } - } - record(completion)?; + })?; } - Ok(newest.map_or(0, |(slot, _)| (slot + 1) % COMPUTE_COMPLETION_SLOTS)) + Ok(()) } fn object(&self, index: usize) -> Result<&KernelObject> { @@ -834,10 +687,6 @@ impl InitData { } /// Constructs the graph after the firmware has published the high-root entries. - #[expect( - clippy::too_many_arguments, - reason = "The firmware graph builder keeps its independent input objects explicit." - )] pub(super) fn build( &mut self, dev: &AsahiDrmDevice, @@ -845,12 +694,7 @@ impl InitData { uat: &mmu::Uat, render: &mmu::Vm, metrics: &KernelObject, - clusters: u32, - active_cores: u32, - variant: &super::fw::Variant, ) -> Result { - self.pm_metrics_va = cfg.pm_metrics.va; - self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { dev, @@ -859,13 +703,9 @@ impl InitData { render, base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, - clusters, }; - if variant.graph == super::fw::variant::InitGraph::V25G83 { - return self.build_25g83(&env, cfg, metrics, active_cores); - } - self.map_register_windows(&env, cfg.windows)?; - let tables = self.build_descriptor_tables(&env, abi::private::SIZE)?; + self.map_register_windows(&env)?; + let tables = self.build_descriptor_tables(&env)?; self.build_render_free_lists(&env)?; let first_block = self.build_render_pool(&env)?; self.build_compute_pools(&env, first_block)?; @@ -873,41 +713,31 @@ impl InitData { self.initialize_firmware_objects(&env, cfg, &tables, &bundle) } - fn map_register_windows( - &mut self, - env: &BuildEnv<'_>, - windows: &[abi::RegisterWindow], - ) -> Result { - for window in windows { - for instance in 0..window.instances { - let (phys, va, size) = window.mapping(instance).ok_or(EINVAL)?; - self.aliases.push( - env.vm.map_io( - env.dev.as_ref().as_ref(), - va, - phys.try_into()?, - size.try_into()?, - mmu::PROT_FW_MMIO_RW, - )?, - GFP_KERNEL, - )?; - } + fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { + for window in &t8140::REGISTER_WINDOWS { + let (phys, va, size) = window.mapping().ok_or(EINVAL)?; + self.aliases.push( + env.vm.map_io( + env.dev.as_ref().as_ref(), + va, + phys.try_into()?, + size.try_into()?, + mmu::PROT_FW_MMIO_RW, + )?, + GFP_KERNEL, + )?; } Ok(()) } /// The private cluster, zero page and the three descriptor tables with their low aliases. - fn build_descriptor_tables( - &mut self, - env: &BuildEnv<'_>, - cluster_size: usize, - ) -> Result { + fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { use mmu::{ PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; - self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? @@ -915,8 +745,7 @@ impl InitData { table.init(); Ok(()) })?; - let page_pool = - self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(page_pool)? .initialize::(0, |table| { table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); @@ -1000,11 +829,7 @@ impl InitData { fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { use mmu::PROT_GPU_SHARED_RW as GS; use CpuMap::WriteBack as WB; - let pages = self.add(KernelObject::backing( - env.dev, - low::RENDER_PAGE_LIST_SIZE, - WB, - )?)?; + let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; self.object_mut(pages)? .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { abi::fill_page_list( @@ -1012,18 +837,10 @@ impl InitData { (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), ) })?; - let runs = self.add(KernelObject::backing( - env.dev, - low::RENDER_RUN_LIST_SIZE, - WB, - )?)?; + let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; let first_block = self.objects.len(); for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing( - env.dev, - abi::FREE_LIST_BLOCK_SIZE, - WB, - )?)?; + self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; } self.render_pages = pages; self.render_runs = runs; @@ -1031,12 +848,7 @@ impl InitData { self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias( - first_block + block, - env.render, - low::render_block(block), - GS, - )?; + self.alias(first_block + block, env.render, low::render_block(block), GS)?; } Ok(first_block) } @@ -1073,8 +885,12 @@ impl InitData { .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; - let ready = - self.add(env.fixed(high::COMPUTE_READY_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; + let ready = self.add(env.fixed( + high::COMPUTE_READY_FREE_LIST, + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; self.object_mut(ready)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -1123,12 +939,12 @@ impl InitData { low_va: 0, high_va: 0, }; 2]; - for (slot, completion_ring) in completion_rings.iter_mut().enumerate() { + for slot in 0..2 { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; self.completion_rings[slot] = ring; self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; - *completion_ring = abi::CompletionRing { + completion_rings[slot] = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], high_va: env.base + offset, }; @@ -1172,13 +988,13 @@ impl InitData { bundle: &BundleObjects, ) -> Result { let base = env.base; - let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; + let opps = Self::opps(env.dev)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, - windows: cfg.windows, - reserved_slots: cfg.reserved_slots, + windows: &t8140::REGISTER_WINDOWS, + reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, opps: &opps, - perf: cfg.perf, + perf: &t8140::PERF, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1186,7 +1002,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(cfg.region_c); + region.init(); Ok(()) })?; self.object_mut(self.cluster)? @@ -1234,8 +1050,8 @@ impl InitData { let slot = role as usize; let main_offset = abi::bundle::MAIN_CONFIG[slot]; let main_va = base + high::BUNDLE + main_offset as u64; - let state_va = base + high::PRIVATE_CLUSTER + self.state_offset(role) as u64; - let status_va = base + high::PRIVATE_CLUSTER + self.status_offset(role) as u64; + let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; let args = abi::MainConfigArgs { bundle_va: base + high::BUNDLE, main_config_va: main_va, @@ -1291,7 +1107,7 @@ impl InitData { (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), ] { self.object(self.cluster)? - .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .store(value, Ordering::Relaxed); } } @@ -1334,7 +1150,7 @@ impl InitData { )?; } mappings.push( - metrics.map_alias(vm, self.pm_metrics_va, mmu::PROT_GPU_FW_SHARED_RW)?, + metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, GFP_KERNEL, )?; Ok(mappings) @@ -1364,69 +1180,37 @@ impl InitData { .load(Ordering::Relaxed)) } - /// The GPU's operating points: `count` of them, each with one voltage cell for the whole - /// GPU or one per cluster. - pub(super) fn opps( - dev: &AsahiDrmDevice, - count: usize, - clusters: u32, - ) -> Result> { - let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; - if opps.len() != count { - return Err(EINVAL); - } - let cells = opps.first().map_or(0, |opp| opp.cells); - if cells != 1 && u32::try_from(cells) != Ok(clusters) { - dev_err!( - dev.as_ref(), - "GPU operating points have {} voltage cells, expected 1 or {}\n", - cells, - clusters - ); - return Err(EINVAL); - } - Ok(opps) - } - - /// The operating points of the table that the GPU node's property `name` names; all have - /// the same number of voltage cells. - fn opp_table(dev: &AsahiDrmDevice, name: &kernel::str::CStr) -> Result> { + fn opps(dev: &AsahiDrmDevice) -> Result> { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let table = node.parse_phandle(name, 0).ok_or(EIO)?; + let table = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EIO)?; let mut opps = KVec::new(); for opp in table.children() { - // One cell for the whole GPU, or one per cluster. - let cells: KVec = opp.get_property(c_str!("opp-microvolt"))?; - if cells.is_empty() || cells.len() > abi::OPP_CELLS { - return Err(EINVAL); - } - let mut volt_uv = [0; abi::OPP_CELLS]; - volt_uv[..cells.len()].copy_from_slice(&cells); opps.push( abi::Opp { freq_hz: opp.get_property(c_str!("opp-hz"))?, - volt_uv, - cells: cells.len(), + volt_uv: opp.get_property(c_str!("opp-microvolt"))?, }, GFP_KERNEL, )?; } - if opps.iter().any(|opp| opp.cells != opps[0].cells) { + if opps.len() != t8140::OPPS { return Err(EINVAL); } Ok(opps) } - fn state_offset(&self, role: Role) -> usize { + fn state_offset(role: Role) -> usize { match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, } } - fn status_offset(&self, role: Role) -> usize { + fn status_offset(role: Role) -> usize { match role { - Role::Primary => self.status_a, + Role::Primary => abi::private::PRIMARY_STATUS_A, Role::Secondary => abi::private::SECONDARY_STATUS_A, } } @@ -1434,7 +1218,7 @@ impl InitData { /// Recheck the host-authored event pointer pair after a bounded number of drains. /// The device mutex serializes the cache and all event consumption. pub(super) fn validate_event_ring(&self, role: Role) -> Result { - let status = self.object(self.cluster)?.gpu_va() + self.status_offset(role) as u64; + let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; let expected = ( status + abi::status::EVENT_STATE, status + abi::status::EVENT_RING, @@ -1463,17 +1247,12 @@ impl InitData { Ok(()) } - /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). - pub(super) fn init_accepted(&self) -> Result { - Ok(self - .status_word(Role::Primary, 0x14)? - .load(Ordering::Acquire) - == 1) - } - pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { - self.object(self.cluster)? - .word(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) + self.object(self.cluster)?.word( + Self::status_offset(role) + .checked_add(offset) + .ok_or(EINVAL)?, + ) } pub(super) fn status_read_words( @@ -1481,8 +1260,11 @@ impl InitData { role: Role, offset: usize, ) -> Result<[u32; N]> { - self.object(self.cluster)? - .read_words(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) + self.object(self.cluster)?.read_words( + Self::status_offset(role) + .checked_add(offset) + .ok_or(EINVAL)?, + ) } pub(super) fn control_counters(&self) -> Result<[u32; 3]> { @@ -1554,7 +1336,7 @@ impl InitData { for offset in [0, 0x10, abi::ChannelState::PRODUCER] { if self .object(self.cluster)? - .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::SeqCst) != abi::OPENING_PRODUCER { @@ -1569,55 +1351,39 @@ impl InitData { /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; + let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; - // Separately allocated views live at offset 0 of their own object. - let (object, base) = match self.views { - Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), - None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), - }; - let end = offset - .checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) + let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { return Err(EINVAL); } let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) - + view * size_of::(); + + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; - if actual != object.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { return Err(EINVAL); } - object.read_words(base + offset) + bundle.read_words(base + offset) } } impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { - let mask = self - .object(self.cluster)? - .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + let mask = self.object(self.cluster)? + .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - // Progress is independently available even when this firmware's key - // address is unknown. Zero keeps ownership unknown and cannot override - // firmware blame, but preserves the progress evidence for diagnostics. - let key = super::recovery::slot_key(slot, self.recovery_slot_keys).unwrap_or(0); - let monitor = self - .recovery_view_words::<6>( - abi::recovery::PROGRESS_VIEW, - abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE, - ) - .ok() - .map(|words| words[0])?; + let key = super::recovery::slot_key(slot)?; + let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, + abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) + .ok().map(|words| words[0])?; Some((key, monitor)) }); - sources.reason = self - .recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .ok() - .map(|words| words[0]); + sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) + .ok().map(|words| words[0]); Ok(sources) } @@ -1625,21 +1391,21 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1651,21 +1417,17 @@ impl super::recovery::Memory for InitData { } fn clear_timestamps(&self) -> Result { - // The per-QID timestamp table is view 4; separately allocated views live at offset 0 of - // their own object. - let (table, offset) = match self.views { - Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), - None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), - }; + let bundle = self.object(self.bundle)?; // Validate the complete table before publishing its first live word. const ENTRIES: usize = 128; const STRIDE: usize = 0x10; - table.pointer(offset, ENTRIES * STRIDE)?; + let offset = abi::bundle::VIEWS[4]; + bundle.pointer(offset, ENTRIES * STRIDE)?; for qid in 0..ENTRIES { - table + bundle .word(offset + qid * STRIDE)? .store(0, Ordering::Relaxed); - table + bundle .dword(offset + qid * STRIDE + 8)? .store(0, Ordering::Relaxed); } @@ -1673,22 +1435,17 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entry(&self, index: usize) -> Result<(u32, u32)> { - if index >= super::recovery::RECOVERY_RECORDS { - return Err(EINVAL); - } + fn entries(&self) -> Result { let cluster = self.object(self.cluster)?; - let offset = self.status_b - + abi::PrimaryStatusB::RECOVERY_INFO - + index * core::mem::size_of::(); + let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; fence(Ordering::Acquire); - let flags = cluster - .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? - .load(Ordering::Relaxed); - let stamp = cluster - .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, stamp))? - .load(Ordering::Relaxed); - Ok((flags, stamp)) + let mut entries = super::recovery::Entries::default(); + for index in 0..256 { + let offset = base + index * core::mem::size_of::(); + let flags = cluster.word(offset)?.load(Ordering::Relaxed); + entries.include(flags); + } + Ok(entries) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs deleted file mode 100644 index 670f4b96dcb76..0000000000000 --- a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs +++ /dev/null @@ -1,230 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! The initialization graph of the 25G83 firmware build. -//! -//! One coprocessor carries both roles: there is one main configuration, with the -//! secondary-only words, and one merged root. The cluster, the hardware data, the power -//! configuration, region C, the views and the firmware area follow the 25G83 layouts -//! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. - -use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; -use crate::g17::{calibration::Calibration, Config, Role}; -use crate::mmu; -use abi::v25g83 as v; -use core::sync::atomic::{fence, Ordering}; -use kernel::{c_str, prelude::*}; - -/// Object indices of the 25G83-specific objects. -struct Objects { - views: [usize; 5], - firmware_area: usize, - power_state: usize, - aon: usize, -} - -impl InitData { - /// Constructs the 25G83 graph after the firmware has published its root entries. - pub(super) fn build_25g83( - &mut self, - env: &BuildEnv<'_>, - cfg: &Config, - metrics: &KernelObject, - active_cores: u32, - ) -> Result { - self.status_a = v::private::STATUS_A; - self.status_b = v::private::STATUS_B; - self.map_register_windows(env, cfg.windows)?; - let tables = self.build_descriptor_tables(env, v::private::SIZE)?; - self.build_render_free_lists(env)?; - let first_block = self.build_render_pool(env)?; - self.build_compute_pools(env, first_block)?; - let bundle = self.build_bundle(env, cfg, metrics)?; - let objects = self.build_25g83_objects(env)?; - self.views = Some(objects.views); - self.initialize_25g83(env, cfg, active_cores, &tables, &bundle, &objects)?; - if cfg.uat_quirks.clean_tables { - // Write back every host-built byte before the firmware may read it. - for object in self.objects.iter_mut() { - let bytes = object.bytes_mut(); - crate::pgtable::clean_poc(bytes.as_ptr(), bytes.len()); - } - crate::pgtable::poc_barrier(); - } - Ok(()) - } - - /// The views, the firmware area, the power-state object and the AON table. - fn build_25g83_objects(&mut self, env: &BuildEnv<'_>) -> Result { - use mmu::{PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS}; - use CpuMap::WriteCombined as WC; - let mut views = [0; 5]; - for (view, offset) in views.iter_mut().zip(v::kernel_window::VIEWS) { - *view = self.add(env.fixed(offset, mmu::UAT_PGSZ, FP, WC)?)?; - } - let firmware_area = - self.add(env.fixed(v::kernel_window::FIRMWARE_AREA, mmu::UAT_PGSZ, FP, WC)?)?; - let power_state = - self.add(env.fixed(v::kernel_window::POWER_STATE, mmu::UAT_PGSZ, FS, WC)?)?; - let aon = self.add(env.fixed(v::kernel_window::AON_TABLE, mmu::UAT_PGSZ, FP, WC)?)?; - Ok(Objects { - views, - firmware_area, - power_state, - aon, - }) - } - - /// Writes the 25G83 firmware objects using the chip calibration, then - /// stages the opening control-ring state. - fn initialize_25g83( - &mut self, - env: &BuildEnv<'_>, - cfg: &Config, - active_cores: u32, - tables: &DescriptorTables, - bundle: &BundleObjects, - objects: &Objects, - ) -> Result { - let bundle_va = self.object(self.bundle)?.gpu_va(); - let cluster_va = self.object(self.cluster)?.gpu_va(); - let config_va = cluster_va + v::private::CONFIG as u64; - let status_a_va = cluster_va + v::private::STATUS_A as u64; - let status_b_va = cluster_va + v::private::STATUS_B as u64; - let aon_va = self.object(objects.aon)?.gpu_va(); - let power_state_va = self.object(objects.power_state)?.gpu_va(); - let firmware_area_va = self.object(objects.firmware_area)?.gpu_va(); - let mut view_va = [0; 5]; - for (va, index) in view_va.iter_mut().zip(objects.views) { - *va = self.object(index)?.gpu_va(); - } - - let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; - let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; - let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; - let calibration = Calibration::read(env.dev)?; - calibration.check_opps(&opps)?; - let hw_args = abi::HwDataArgs { - chip_id: cfg.chip_id, - windows: cfg.windows, - reserved_slots: cfg.reserved_slots, - opps: &opps, - perf: &calibration.perf, - completion_rings: bundle.completion_rings, - qos_va: self.object(bundle.qos)?.gpu_va(), - }; - let hw_args_25g83 = v::HwDataArgs25G83 { - active_cores, - cs_opps: &cs_opps, - afr_opps: &afr_opps, - }; - // SAFETY: Plain C entropy source. - let entropy = unsafe { kernel::bindings::get_random_u32() }; - self.object_mut(self.bundle)? - .initialize::(0, |hw| { - hw.init(&hw_args, &hw_args_25g83)?; - Ok(()) - })?; - self.object_mut(self.bundle)? - .initialize::(v::POWER_CONFIG, |power| { - power.init( - config_va, - aon_va, - entropy, - calibration.max_power_mw, - &calibration.leakage, - ); - Ok(()) - })?; - self.object_mut(self.region_c)? - .initialize::(0, |region| { - region.init(cfg.region_c); - region.max_power_mw = calibration.max_power_mw; - Ok(()) - })?; - for (index, word) in v::AON_TABLE.iter().enumerate() { - self.object_mut(objects.aon)?.write(index * 4, *word)?; - } - - // Status A and B: only the boot words and the firmware-control pointers. - self.object_mut(self.cluster)? - .write(v::private::STATUS_A, abi::StatusBlock::new())?; - self.object_mut(self.cluster)? - .write(v::private::STATUS_B, abi::StatusBlock::new())?; - // Status B keeps the A18 Pro's field offsets up to its recovery words. - let fwctl_va = self.object(tables.fwctl)?.gpu_va(); - let fwctl = - v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); - let cluster = self.object(self.cluster)?; - cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); - cluster - .dword(fwctl + 8)? - .store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); - - // One main configuration carrying both roles. - let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; - let main_va = bundle_va + main_offset as u64; - let args = abi::MainConfigArgs { - bundle_va, - main_config_va: main_va, - state_grid_va: cluster_va + self.state_offset(Role::Primary) as u64, - status_a_va, - pb_descriptors_low_va: low::PB_DESCRIPTORS, - page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, - }; - let views = abi::PrimaryViews { - zero_page_va: self.object(bundle.zero)?.gpu_va(), - pb_descriptors_va: self.object(tables.pb)?.gpu_va(), - page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), - }; - self.object_mut(self.bundle)? - .initialize::(main_offset, |main| { - main.init_primary(&args, &views); - main.view_va = view_va; - main.firmware_area_va = [firmware_area_va; 2]; - main.unk_18 = (cluster_va + v::private::MAIN_HEADER as u64).to_le_bytes(); - // The secondary role's words, on the single instance. - main.unk_300 = 4; - main.power_config_va = bundle_va + v::POWER_CONFIG as u64; - main.unk_32c = 0; - main.unk_344 = 0; - Ok(()) - })?; - self.object_mut(self.bundle)?.write( - main_offset + abi::MainConfig::CONTROL_RING, - abi::OpeningRecord::new(abi::Role::Primary), - )?; - - // The merged root. - let root = abi::RootArgs { - region_a_va: self.object(bundle.region_a)?.gpu_va(), - region_c_va: self.object(self.region_c)?.gpu_va(), - main_config_va: main_va, - status_a_va, - ias: cfg.uat_ias, - oas: cfg.uat_oas, - }; - self.object_mut(self.roots)?.write( - 0, - abi::SecondaryRoot::merged(&root, status_b_va, power_state_va, config_va), - )?; - for &(offset, value, repeat) in v::POWER_STATE_WORDS { - for index in 0..repeat { - self.object_mut(objects.power_state)? - .write(offset + 4 * index, value)?; - } - } - - // The opening control record is staged in slot 0. - for (offset, value) in [ - (0, 0), - (0x10, 0), - (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), - ] { - self.object(self.cluster)? - .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? - .store(value, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 7e43c163ac4a7..b5a5b3759de6d 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -13,11 +13,8 @@ use kernel::{ sync::Arc, }; -pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; -/// One logical render queue owns one physical pair, whose active ring holds -/// `RENDER_DEPTH` commands. A deeper scheduler would only park run_job in -/// admission until a retirement; the scheduler waits for the credit instead. -const RENDER_CREDITS: u32 = crate::hw::t8140::queues::RENDER_DEPTH as u32; +pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; +const RENDER_CREDITS: u32 = 32; const COMPUTE_CREDITS: u32 = 16; #[derive(Clone, Copy, PartialEq, Eq)] @@ -146,49 +143,29 @@ impl Packet { } } - /// Replay holds the device mutex. Its original inputs were satisfied before - /// first publication; never sleep on a fence that needs event service here. - pub(crate) fn check_replay_dependencies(&self) -> Result { - for fence in &self.inputs { - super::dependency::replay_status(fence_status(fence))?; - } - for fence in &self.firmware { check_status(fence)?; } - self.completion.check_source_inputs()?; - Ok(()) - } - /// Scheduler dependency readiness includes failed fences. Consume the actual /// status before publication, and retain the references through that check. pub(crate) fn check_dependencies(&self) -> Result { - let polls = if self.command.long_job() { 5 } else { 1 }; - let deadline = super::now_ns().saturating_add(u64::from(TIMEOUT_MS) * polls * 1_000_000); for fence in &self.inputs { - loop { - if self.cancel_requested() || self.completion.status().get() != 0 { - return Err(ECANCELED); - } - if fence_status(fence) != 0 { break; } - if super::now_ns() >= deadline { return Err(ECANCELED); } - // SAFETY: The retained fence owns the pointer throughout the bounded wait. - let waited = unsafe { - bindings::dma_fence_wait_timeout( - fence.raw(), - true, - kernel::time::msecs_to_jiffies(10) as isize, - ) - }; - // An interrupted wait is not a producer failure. - if waited < 0 { - return Err(Error::from_errno(waited as i32)); - } - if waited > 0 { break; } + // SAFETY: The retained fence owns the pointer throughout the wait and read. + let waited = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + true, + kernel::task::MAX_SCHEDULE_TIMEOUT, + ) + }; + if waited < 0 { + return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + } + if waited == 0 { + return Err(ECANCELED); } check_status(fence)?; } for fence in &self.firmware { check_status(fence)?; } - self.completion.check_source_inputs()?; Ok(()) } @@ -202,13 +179,6 @@ impl Packet { .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .is_ok() } - /// Gives up a claim taken from the submitting ioctl before the job was - /// pushed: nothing became visible, and the scheduler claims it again. - pub(crate) fn abandon_publication(&self) { - let _ = self - .publication - .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Acquire); - } pub(crate) fn mark_published(&self) { self.publication.store(2, Ordering::Release); } @@ -231,7 +201,7 @@ impl Packet { } pub(crate) fn set_early_result(&self, result: Result) { self.early.store( - result.map_or_else(|error| error.to_errno(), |()| 1), + result.map_or_else(|error| error.to_errno(), |_| 1), Ordering::Release, ); } @@ -258,7 +228,7 @@ pub(crate) fn fence_status(fence: &Fence) -> i32 { fn check_status(fence: &Fence) -> Result { let status = fence_status(fence); if status < 0 { - Err(super::dependency::PRODUCER_FAILED) + Err(super::dependency::inherited_error(Error::from_errno(status))) } else { Ok(()) } @@ -268,15 +238,11 @@ fn check_status(fence: &Fence) -> Result { /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { /// Device detector retained by render completions after queue teardown. - fn input_sources(&self, _inputs: &[crate::file::SyncItem]) - -> Option> { None } - fn feed(&self) -> Option> { None } - /// Validate compute sizing without reserving a physical channel. Queued - /// commands may wait on a host signal submitted after this ioctl returns. + /// Lazily create or rebind this logical queue's physical compute graph. fn ensure_compute(&self, scs_pages: u16) -> Result; fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { if pages != [0; 2] { return Err(EINVAL); } @@ -286,16 +252,12 @@ pub(crate) trait Backend: Send + Sync { fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. fn release_owner(&self); - /// A fence to wait for before `publish`, signalled by the release of the capacity the - /// packet lacks. Called again once it signals. - fn prepare(&self, packet: &Arc) -> Option; + fn prepare(&self, packet: &Packet) -> Option; fn publish(&self, packet: Arc) -> Result; fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); /// Transfer a timeout to an outstanding off-lock graph transaction, if any. fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; - /// A long job of another queue is running beside `packet`. - fn long_job_beside(&self, _packet: &Arc) -> bool { false } /// Return true only after taking publication ownership and recording its result. fn try_early(&self, packet: &Arc) -> bool; fn count_backlog(&self, packet: &Packet); @@ -309,8 +271,6 @@ struct Job { struct Timeout { backend: Arc, victim_renewals: u8, - execution_renewals: u8, - shared_renewals: u8, packet: Arc, } @@ -327,15 +287,12 @@ impl Drop for Job { impl sched::JobImpl for Job { type TimeoutData = Timeout; - const MODULE: Option<&'static kernel::ThisModule> = - Some(kernel::module::this_module::()); + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); fn timeout_data(&self) -> Timeout { Timeout { backend: self.backend.clone(), victim_renewals: 0, - execution_renewals: 0, - shared_renewals: 0, packet: self.packet.clone(), } } @@ -344,37 +301,18 @@ impl sched::JobImpl for Job { if finished || job.packet.completion.take_replay_timeout() { return true; } - if super::timeout::renew_execution( - job.packet.is_published(), job.packet.command.long_job(), &mut job.execution_renewals, - ) { - return true; - } - // Other work retiring means the GPU is shared, not hung. So does another long job - // running beside a long job, though neither retires until it ends. - let shared = super::timeout::retiring(super::recovery::since_retired(super::now_ns())) - || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet)); - if super::timeout::renew_shared(job.packet.is_published(), shared, &mut job.shared_renewals) { - return true; - } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); - if deferred { - job.victim_renewals = job.victim_renewals.saturating_add(1); + if deferred && job.packet.engine() == Engine::Compute { + job.victim_renewals += 1; } deferred } fn prepare(job: &mut Self) -> Option { - super::trace::schedule(&job.packet, 5, 0); - let wait = job.backend.prepare(&job.packet); - super::trace::schedule(&job.packet, 6, i32::from(wait.is_some())); - wait + job.backend.prepare(&job.packet) } fn run(job: &mut Self) -> Result> { - super::trace::schedule(&job.packet, 3, 0); - let result = job.backend.publish(job.packet.clone()).map(Some); - super::trace::schedule(&job.packet, 4, - result.as_ref().err().map_or(0, |error| error.to_errno())); - result + job.backend.publish(job.packet.clone()).map(Some) } fn timed_out(job: &mut Timeout, finished: bool) -> sched::Status { // The scheduler retains the detached job throughout this callback. Keep @@ -402,32 +340,13 @@ pub(crate) struct Fences { pub(crate) ready: Fence, pub(crate) completed: Fence, pub(crate) publication: Option>, - pub(crate) owner: Arc, -} - -impl super::dependency::Retention for Fences { - fn retained(&self) -> bool { - self.owner.retains_vm() - } -} - -impl Fences { - /// Scheduler::enqueue always supplies our explicit completion fence here. - /// It has no polling callback, so pending frontier checks need no lock. - pub(crate) fn status(&self) -> i32 { - unsafe extern "C" { - fn asahi_g17_completion_fence_status(fence: *mut bindings::dma_fence) -> i32; - } - // SAFETY: completed owns the live private fence throughout this read. - unsafe { asahi_g17_completion_fence_status(self.completed.raw()) } - } } impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { - Engine::Render => (RENDER_CREDITS, c_str!("aurora_g17_render")), - Engine::Compute => (COMPUTE_CREDITS, c_str!("aurora_g17_compute")), + Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), + Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), }; let scheduler = sched::Scheduler::new(dev.as_ref().as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; @@ -445,7 +364,6 @@ impl Scheduler { early: bool, publication: Option, ) -> Result { - super::trace::schedule(&packet, 0, dependencies.len() as i32); let completed = packet.completion.fence(); let host_timestamps = packet.completion.has_timestamps(); // Only commands which actually write timestamps become host-prefix @@ -478,15 +396,10 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - let inline = early && self.backend.try_early(&packet); - super::trace::schedule(&packet, if inline { 1 } else { 2 }, 0); - if !inline { - if packet.completion.needs_source_ready() { - packet.completion.set_source_ready(ready.clone()); - } + if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { self.backend.count_backlog(&packet); } job.push(); - Ok(Fences { ready, completed, publication: host_publication, owner: packet.completion.clone() }) + Ok(Fences { ready, completed, publication: host_publication }) } } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 6499cae106309..3b78b9443a86d 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,15 +6,8 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::{ - mem::ManuallyDrop, - sync::atomic::{fence, Ordering}, -}; -use kernel::{ - new_mutex, - prelude::*, - sync::{Arc, Mutex}, -}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; use super::{ fw::{kick::*, queue::DataMaster}, @@ -24,40 +17,8 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// Render identities that must stay reachable for `queues` logical queues: -/// one tiling/fragment pair each, plus the two bootstrap reservations. -/// -/// Installed identities never change engine before processor stop, so a -/// compute channel permanently narrows what render can still install. Compute -/// may take a free QID only while every open logical queue, and -/// [`RENDER_HEADROOM`] queues opened later, can still install a dedicated -/// pair; logical-queue admission refuses a queue whose pair no longer fits. -pub(super) const fn render_reserve(queues: usize) -> usize { - let queues = queues + RENDER_HEADROOM; - let max = crate::mmu::MAX_EXECUTION_CONTEXTS as usize; - 2 * if queues < max { queues } else { max } + 2 -} -/// Logical queues that may still open after compute growth without losing a -/// dedicated render pair. -const RENDER_HEADROOM: usize = 24; -/// The render reserve with every execution context open. -const FULL_RENDER_RESERVE: usize = render_reserve(crate::mmu::MAX_EXECUTION_CONTEXTS as usize); -/// Compute channels that stay reachable with every execution context open. -#[cfg(CONFIG_KUNIT = "y")] -pub(super) const COMPUTE_FLOOR: usize = QID_COUNT - FULL_RENDER_RESERVE; -/// Upper bound on distinct physical compute channels named by one kick. -pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT; -/// Maximum logical queues after firmware-owned QIDs are excluded. -pub(super) const fn queue_limit(floor: usize) -> u32 { - ((FULL_RENDER_RESERVE - 2 - floor) / 2) as u32 -} - -const _: () = { - assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); - // Compute keeps this many channels even with every execution context open. - assert!(QID_COUNT - FULL_RENDER_RESERVE >= 4); - assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); -}; +/// QIDs kept free for render pairs when compute queues fill the table. +const COMPUTE_RENDER_RESERVE: usize = 16; /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -86,40 +47,19 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], - initial_render: [Option; 2], + compute_publication: u64, leased: usize, - /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). - floor: usize, } impl QueueIds { - pub(crate) fn new(floor: usize) -> impl Init { + pub(crate) fn new() -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), - initial_render: [None; 2], + compute_publication: 0, leased: 0, - // The reserve must keep render capacity above the floor. - floor: if floor < FULL_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } - /// Keep the initial render reservations until another engine needs their - /// capacity. No graph receives these tokens, so an unpublished reservation - /// can be replaced without changing an installed physical channel. - pub(crate) fn reserve_initial_render(&mut self) -> Result { - if self.leased != 0 { - return Err(EINVAL); - } - for (qid, engine) in [DataMaster::Tiling, DataMaster::Fragment] - .into_iter() - .enumerate() - { - let id = self.reserve(qid as u64 + 1, engine, Some(qid as u8))?; - self.initial_render[qid] = Some(id); - } - Ok(()) - } - /// Reuses only an exact owner/engine match. Preferred placement is for /// bootstrapping; it does not reserve a partition for an engine. pub(crate) fn reserve( @@ -127,58 +67,6 @@ impl QueueIds { owner: u64, engine: DataMaster, preferred: Option, - ) -> Result { - self.reserve_for(owner, engine, preferred, crate::mmu::MAX_EXECUTION_CONTEXTS as usize) - } - - fn render_ids(&self) -> usize { - self.entries - .iter() - .flatten() - .filter(|entry| matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment)) - .count() - } - - /// Whether `queues` open logical queues can each still install a dedicated - /// render pair, beside the bootstrap reservations and installed compute. - pub(crate) fn render_fits(&self, queues: usize) -> bool { - self.render_ids() + (QID_COUNT - self.floor - self.leased) >= 2 * queues + 2 - } - - /// Whether a render pair not yet built can still reserve both of its identities. - pub(crate) fn render_pair_fits(&self) -> bool { - QID_COUNT - self.floor - self.leased >= 2 - } - - /// Whether a fresh compute identity fits beside the render reserve of - /// `queues` open logical queues, without reserving it. - pub(crate) fn compute_fits(&self, queues: usize) -> bool { - let initial = self.leased == QID_COUNT - self.floor - && self.initial_render.iter().flatten().any(|id| self.can_cancel_unpublished(*id)); - self.compute_room(queues, usize::from(initial)) - } - - fn compute_room(&self, queues: usize, replacement: usize) -> bool { - // Existing render identities already satisfy this reservation. Counting - // them twice can refuse compute even while physical QIDs remain free. - // Replacing an initial reservation consumes one render identity. - // Include that transition when protecting render's minimum, and - // count the replacement as available compute capacity. - let render = self.render_ids(); - QID_COUNT - self.floor - self.leased + replacement - > render_reserve(queues) - .min(FULL_RENDER_RESERVE - self.floor) - .saturating_sub(render - replacement) - } - - /// As [`Self::reserve`], with the compute reserve sized for `queues` open - /// logical queues instead of every possible execution context. - pub(crate) fn reserve_for( - &mut self, - owner: u64, - engine: DataMaster, - preferred: Option, - queues: usize, ) -> Result { if owner == 0 { return Err(EINVAL); @@ -195,36 +83,16 @@ impl QueueIds { Err(EINVAL) }; } - // The QIDs below the floor are never available to the host. - let total = QID_COUNT - self.floor; - let initial = if engine == DataMaster::Compute && self.leased == total { - self.initial_render - .iter() - .copied() - .enumerate() - .find_map(|(slot, id)| { - id.filter(|id| self.can_cancel_unpublished(*id)) - .map(|id| (slot, id)) - }) - } else { - None - }; - // Published identities always retain their original lifetime rules. - if engine == DataMaster::Compute && !self.compute_room(queues, usize::from(initial.is_some())) { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); + // Healthy existing channels keep their QID. Never recycle a published + // identity to recover capacity after a fault; leave eight render pairs. + if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { + return Err(ENOSPC); } let index = preferred - .map(|index| usize::from(index) + self.floor) + .map(usize::from) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| { - self.entries - .iter() - .skip(self.floor) - .position(Option::is_none) - .map(|index| index + self.floor) - }) - .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) - .ok_or_else(|| crate::g17::capacity::refused(crate::g17::capacity::Site::PhysicalQids))?; + .or_else(|| self.entries.iter().position(Option::is_none)) + .ok_or(ENOSPC)?; let id = Id { qid: index as u8, owner, @@ -234,14 +102,16 @@ impl QueueIds { id, published: false, }); - if let Some((slot, _)) = initial { - self.initial_render[slot] = None; - } else { - self.leased += 1; - } + self.leased += 1; Ok(id) } + /// Ordered under the device mutex; no atomic operation on the submission path. + pub(crate) fn next_compute_publication(&mut self) -> Result { + self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; + Ok(self.compute_publication) + } + /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { @@ -257,13 +127,6 @@ impl QueueIds { Ok(()) } - pub(crate) fn can_cancel_unpublished(&self, id: Id) -> bool { - self.entries - .get(usize::from(id.qid)) - .and_then(Option::as_ref) - .is_some_and(|entry| entry.id == id && !entry.published) - } - pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; match slot { @@ -277,117 +140,6 @@ impl QueueIds { } } -#[kunit_tests(asahi_g17_qid_accounting)] -mod accounting_tests { - use super::*; - - fn table() -> KBox { - KBox::init(QueueIds::new(0), GFP_KERNEL).unwrap() - } - - #[test] - fn render_identities_satisfy_the_compute_reserve() { - let mut ids = table(); - for owner in 1..=FULL_RENDER_RESERVE as u64 { - ids.reserve(owner, DataMaster::Tiling, None).unwrap(); - } - for owner in FULL_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { - ids.reserve(owner, DataMaster::Compute, None).unwrap(); - } - assert_eq!(ids.leased, QID_COUNT); - assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); - assert_eq!(ids.leased, QID_COUNT); - } - - #[test] - fn compute_cannot_consume_the_render_minimum() { - let mut ids = table(); - for owner in 1..=(QID_COUNT - FULL_RENDER_RESERVE) as u64 { - ids.reserve(owner, DataMaster::Compute, None).unwrap(); - } - assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); - let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); - assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); - ids.cancel_unpublished(tiling).unwrap(); - assert_eq!(ids.leased, QID_COUNT - FULL_RENDER_RESERVE); - } - - #[test] - fn dynamic_compute_growth_preserves_render_headroom_above_firmware_floor() { - for floor in [0, 4, 32] { - let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); - let queues = 2; - let mut owner = 1; - while ids.compute_fits(queues) { - let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); - assert!(usize::from(id.qid()) >= floor); - ids.publish(id).unwrap(); - owner += 1; - } - assert_eq!(ids.leased, QID_COUNT - floor - render_reserve(queues)); - assert_eq!(ids.reserve_for(owner, DataMaster::Compute, None, queues), Err(ENOSPC)); - assert!(ids.render_fits(queues + RENDER_HEADROOM)); - assert!(!ids.render_fits(queues + RENDER_HEADROOM + 1)); - } - } - - #[test] - fn full_logical_queue_budget_keeps_compute_floor_and_excludes_firmware_ids() { - for floor in [0, 4, 32] { - let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); - let queues = queue_limit(floor) as usize; - for owner in 1..=COMPUTE_FLOOR as u64 { - let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); - assert!(usize::from(id.qid()) >= floor); - ids.publish(id).unwrap(); - } - assert!(!ids.compute_fits(queues)); - assert!(ids.render_fits(queues)); - assert!(!ids.render_fits(queues + 1)); - } - } - - #[test] - fn initial_reservations_yield_but_published_ids_do_not() { - let mut ids = table(); - ids.reserve_initial_render().unwrap(); - for owner in 3..=QID_COUNT as u64 { - ids.reserve(owner, DataMaster::Tiling, None).unwrap(); - } - let first = ids.reserve(1000, DataMaster::Compute, None).unwrap(); - let second = ids.reserve(1001, DataMaster::Compute, None).unwrap(); - assert_ne!(first.qid(), second.qid()); - assert!(ids.initial_render.iter().all(Option::is_none)); - assert_eq!(ids.leased, QID_COUNT); - ids.publish(first).unwrap(); - assert_eq!(ids.cancel_unpublished(first), Err(EINVAL)); - assert_eq!(ids.leased, QID_COUNT); - ids.cancel_unpublished(second).unwrap(); - assert_eq!(ids.leased, QID_COUNT - 1); - assert_eq!(ids.cancel_unpublished(second), Err(EINVAL)); - assert_eq!(ids.leased, QID_COUNT - 1); - } - - #[test] - fn failed_second_render_id_returns_only_the_unpublished_first() { - let mut ids = table(); - for owner in 1..QID_COUNT as u64 { - let id = ids.reserve(owner, DataMaster::Tiling, None).unwrap(); - ids.publish(id).unwrap(); - } - let first = ids.reserve(1000, DataMaster::Tiling, None).unwrap(); - assert_eq!(ids.reserve(1001, DataMaster::Fragment, None), Err(ENOSPC)); - ids.cancel_unpublished(first).unwrap(); - assert_eq!(ids.leased, QID_COUNT - 1); - let replacement = ids.reserve(1002, DataMaster::Compute, None).unwrap(); - assert_eq!(replacement.qid(), first.qid()); - assert_ne!(replacement, first); - assert_eq!(ids.reserve(1002, DataMaster::Compute, None), Ok(replacement)); - assert_eq!(ids.leased, QID_COUNT); - assert_eq!(ids.reserve(1002, DataMaster::Fragment, None), Err(EINVAL)); - } -} - // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { @@ -395,94 +147,10 @@ enum Backing { high: mmu::KernelMapping, low: KernelObject, }, - Render(RenderRing), -} - -/// A render ring's pages and the shared lower reservation clients alias. -struct RenderStorage { - // The lower alias drops before its backing. - low: mmu::KernelMapping, - high: KernelObject, -} - -/// Render ring storage outlives its rings for the life of the device. -/// -/// A client VM keeps each render ring alias it installs until the VM closes, -/// while an installed pair whose identities were never published can be -/// reclaimed and its rings dropped. Freeing that storage would hand its shared -/// lower address to another ring; a client still aliasing the old pages there -/// that later receives the new ring could not alias it (`prepare_client_alias` -/// fails with EFAULT). Retired storage is therefore kept and reused by later -/// rings: an address keeps naming the same pages, so every alias stays valid. -#[pin_data] -pub(crate) struct RenderRings { - #[pin] - stock: Mutex, -} - -struct RingStock { - /// Retired storage; `true` marks storage of a ring that never wrote an - /// entry, which a later ring may reuse. Written rings stay retired. - retired: KVec<(bool, RenderStorage)>, - /// Storage created so far; `retired` keeps capacity for all of it, so a - /// dropped ring always returns its storage without allocating. - created: usize, -} - -impl RenderRings { - pub(crate) fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { - stock <- new_mutex!( - RingStock { retired: KVec::new(), created: 0 }, - "G17 render kick rings" - ), - }), - GFP_KERNEL, - ) - } - - fn take(&self) -> Option { - let mut stock = self.stock.lock(); - let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; - let last = stock.retired.len() - 1; - stock.retired.swap(index, last); - stock.retired.pop().map(|(_, storage)| storage) - } - - /// Reserves the return of one more, just created storage. - fn admit(&self) -> Result { - let mut stock = self.stock.lock(); - let created = stock.created.checked_add(1).ok_or(EOVERFLOW)?; - let additional = created.saturating_sub(stock.retired.len()); - stock.retired.reserve(additional, GFP_KERNEL)?; - stock.created = created; - Ok(()) - } - - fn put(&self, storage: RenderStorage, reusable: bool) { - let mut stock = self.stock.lock(); - if let Err(rejected) = stock.retired.push_within_capacity((reusable, storage)) { - // `admit` reserved room for every created storage. Never free it. - core::mem::forget(rejected); - } - } -} - -/// Storage lent to one render ring; returned to the stock when the ring drops. -struct RenderRing { - storage: ManuallyDrop, - rings: Arc, - /// Set once an entry was written: such storage is never reused. - written: bool, -} - -impl Drop for RenderRing { - fn drop(&mut self) { - // SAFETY: `storage` is taken exactly once, here, and not used afterwards. - let storage = unsafe { ManuallyDrop::take(&mut self.storage) }; - self.rings.put(storage, !self.written); - } + Render { + low: mmu::KernelMapping, + high: KernelObject, + }, } /// An installed queue's permanent ring. Client aliases are held by the bound @@ -518,9 +186,6 @@ impl PreparedKick<'_> { // the new entry starts before the outer command is published. unsafe { self.destination.write_unaligned(self.entry) }; fence(Ordering::SeqCst); - if let Backing::Render(ring) = &mut self.queue.backing { - ring.written = true; - } self.queue.previous = Some(self.timestamp); self.queue.timestamp = self.timestamp.next(); self.timestamp @@ -555,45 +220,14 @@ impl Queue { }) } - /// Retired storage of a never-written ring is reused first. Its address - /// names the same pages in every client that aliases it, so an existing - /// alias in `client` is kept (None) and a missing one is created. pub(crate) fn render( alloc: &Allocator<'_>, id: Id, client: &mmu::Vm, - rings: &Arc, - ) -> Result<(Self, Option)> { + ) -> Result<(Self, mmu::KernelMapping)> { if id.engine == DataMaster::Compute { return Err(EINVAL); } - if let Some(storage) = rings.take() { - let mut ring = RenderRing { - storage: ManuallyDrop::new(storage), - rings: rings.clone(), - written: false, - }; - // A fresh ring is zeroed; no firmware ever read this one. - ring.storage.high.bytes_mut().fill(0); - fence(Ordering::SeqCst); - let address = ring.storage.low.iova(); - let alias = - ring.storage - .high - .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); - // On failure the storage returns to the stock and a fresh ring is built. - if let Ok(alias) = alias { - return Ok(( - Self { - backing: Backing::Render(ring), - id, - timestamp: KickTimestamp::FIRST, - previous: None, - }, - alias, - )); - } - } let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, @@ -621,26 +255,14 @@ impl Queue { Err(error) => return Err(error), } }; - let storage = RenderStorage { low, high }; - // Count the storage only once it exists. Without room to return it, - // nothing has aliased it but `client`, which drops first. - if let Err(error) = rings.admit() { - drop(client); - drop(storage); - return Err(error); - } Ok(( Self { - backing: Backing::Render(RenderRing { - storage: ManuallyDrop::new(storage), - rings: rings.clone(), - written: false, - }), + backing: Backing::Render { low, high }, id, timestamp: KickTimestamp::FIRST, previous: None, }, - Some(client), + client, )) } @@ -650,19 +272,19 @@ impl Queue { pub(crate) fn low_va(&self) -> u64 { match &self.backing { Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render(ring) => ring.storage.low.iova(), + Backing::Render { low, .. } => low.iova(), } } pub(crate) fn firmware_va(&self) -> u64 { match &self.backing { Backing::Compute { high, .. } => high.iova(), - Backing::Render(ring) => ring.storage.high.gpu_va(), + Backing::Render { high, .. } => high.gpu_va(), } } fn object(&mut self) -> &mut KernelObject { match &mut self.backing { Backing::Compute { low, .. } => low, - Backing::Render(ring) => &mut ring.storage.high, + Backing::Render { high, .. } => high, } } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { @@ -675,7 +297,7 @@ impl Queue { pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { Backing::Compute { low, .. } => low, - Backing::Render(ring) => &ring.storage.high, + Backing::Render { high, .. } => high, }; object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) } @@ -731,20 +353,3 @@ impl Queue { }) } } - -impl QueueIds { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - let mut engines = [0usize; 3]; - let mut published = 0; - for entry in self.entries.iter().flatten() { - engines[match entry.id.engine { - DataMaster::Tiling => 0, - DataMaster::Fragment => 1, - DataMaster::Compute => 2, - }] += 1; - published += usize::from(entry.published); - } - writeln!(f, "physical_qids used={} total={} published={} tiling={} fragment={} compute={}", - self.leased, QID_COUNT, published, engines[0], engines[1], engines[2]) - } -} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 1dbfcc8fd3ce5..9da90d2b0843a 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -12,55 +12,39 @@ //! //! # Locking //! -//! `Shared::state` is the device lock. It serializes binding, recovery, the event worker and all -//! firmware-shared state that has no narrower owner. Steady-state compute publication into a -//! bound queue takes only that queue's cell lock and the port's locks ([`port`]), reached under -//! a preparation lease. Completions are signalled after dropping the device lock. Lock order: -//! `Shared::state` -> compute queue cell (one at a time) -> port locks -> availability domains; -//! `Shared::state` -> UAT shared state -> VM exec lock -> GEM reservation. `RoleState::ack` -//! nests inside nothing. Capacity waits block on an availability domain signalled by the -//! release that provides the capacity, never on a sleep. - -mod availability; +//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all +//! firmware-shared state, including the bounded waits for the firmware that publication needs. +//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state +//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. + mod buffer; -mod calibration; mod channel; mod command; pub(crate) use command::Attachments; mod completion; pub(crate) mod context; -mod data_segment; mod dependency; -mod sync_input; -pub(crate) mod sync_output; -mod trace; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; -mod capacity; mod event; mod feed; mod fence; mod freelist; pub(crate) mod fw; -pub(crate) mod hazard; mod ids; +pub(crate) mod hazard; mod initdata; mod job; mod kick; mod object; -mod port; -pub(crate) mod power; +mod power; mod preparation; mod qos; mod queue; mod recovery; -mod replay; -mod reports; mod runtime; -pub(crate) mod stats; pub(crate) mod status; mod teardown; -pub(crate) mod telemetry; mod timeout; mod validation; @@ -68,8 +52,6 @@ use core::any::Any; use core::ops::Range; use core::sync::atomic::{ AtomicBool, - AtomicU32, - AtomicU64, Ordering, // }; @@ -77,12 +59,12 @@ use kernel::{ c_str, device::{self, Core}, drm::gem::shmem, - impl_has_delayed_work, impl_has_work, io::{ IoBase, IoSysMap, Region, // }, + impl_has_delayed_work, impl_has_work, new_condvar, new_delayed_work, new_mutex, new_work, platform, prelude::*, soc::apple::rtkit, @@ -167,64 +149,12 @@ pub(crate) struct Config { pub(crate) uat_ias: u32, /// Output (physical) address bits of the UAT. pub(crate) uat_oas: u32, - /// Offset of the secondary coprocessor's kernel root in the `pagetables` region; `None` for - /// a single-instance firmware. - pub(crate) secondary_root_offset: Option, - /// Where the coprocessors' crash log buffers live. - pub(crate) crash_buffers: CrashBuffers, + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. + pub(crate) secondary_root_offset: u64, + /// Crash log buffer of each [`Role`]. + pub(crate) crash_buffers: [FixedBuffer; 2], /// Parameter buffer page metrics shared with the firmware. pub(crate) pm_metrics: FixedBuffer, - /// Register windows the firmware maps through the hardware data. - pub(crate) windows: &'static [fw::initdata::RegisterWindow], - /// Register-map slots that are reserved but have no window. - pub(crate) reserved_slots: &'static [usize], - /// Per-SoC parts of the performance tables. - pub(crate) perf: &'static fw::initdata::PerfConfig, - /// Number of operating points the device tree lists. - pub(crate) opps: usize, - /// UAT policy where it differs from the A18 Pro's. - pub(crate) uat_quirks: mmu::UatQuirks, - /// Snapshot the firmware root's own top-level entries after the firmware booted, clear the - /// others and confirm the snapshot after building (A18 Pro). Otherwise the root is used as - /// the firmware left it, which is the only way the M4 Pro has run. - pub(crate) adopt_root_entries: bool, - /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware - /// starts (A18 Pro). The M4 Pro has only run with neither written. - pub(crate) fabric_bridge: bool, - /// The SoC's values of region C. - pub(crate) region_c: &'static fw::initdata::RegionCConfig, - /// Physical address of the firmware's per-slot recovery keys, read while the firmware is - /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the - /// firmware's own blame). - pub(crate) recovery_slot_keys: Option, - /// GPU generation reported to userspace. - pub(crate) gpu_generation: hw::GpuGen, - /// The firmware builds this SoC runs. - pub(crate) firmware: fw::variant::Firmware, - /// Layouts of the command buffers that follow the GPU's core layout. - pub(crate) core_layout: CoreLayout, -} - -/// Layouts of the firmware command buffers that follow a SoC's GPU core layout. -pub(crate) struct CoreLayout { - /// The measured cluster and enabled core counts; `None` admits every configuration. - pub(crate) cores: Option<(u32, u32)>, - /// An experimental layout: core slots and enabled cores per cluster. - pub(crate) alternate_core_masks: Option<(u32, u32)>, - /// A compute queue's operand-state buffer. - pub(crate) operand_state: fw::compute::OperandState, - /// A render's deflake buffer. - pub(crate) deflake: fw::render::Deflake, -} - -/// Where a G17 target's crash log buffers live. -pub(crate) enum CrashBuffers { - /// One buffer per [`Role`] that the firmware requests at a fixed address of the kernel - /// lower root, mapped by the driver before the coprocessors start. - Fixed([FixedBuffer; 2]), - /// The firmware places its crash log in its own data segment, the `fw-data` reserved - /// region, and asks for it there by physical address. - FirmwareData, } /// A firmware coprocessor instance. @@ -267,9 +197,6 @@ const RECEIVER_RETRY_MS: i64 = 20; /// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; -/// One-millisecond polls for a firmware that acknowledges its initialization only in status A -/// ([`fw::variant::InitAck::Status`]). -const INIT_ACCEPT_POLLS: usize = 3000; /// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in /// its buffer requests. @@ -280,15 +207,7 @@ const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; /// A crash log buffer handed to RTKit. pub(crate) struct CrashLog { iova: usize, - map: CrashLogMap, -} - -/// The CPU mapping of a crash log buffer. -enum CrashLogMap { - /// The driver's buffer ([`CrashBuffers::Fixed`]). - Object(shmem::VMapOwned), - /// A buffer in the firmware's data segment ([`CrashBuffers::FirmwareData`]). - FirmwareData(data_segment::DataSegmentLog), + map: shmem::VMapOwned, } impl rtkit::Buffer for CrashLog { @@ -297,10 +216,7 @@ impl rtkit::Buffer for CrashLog { } fn buf(&mut self) -> Result> { - match &mut self.map { - CrashLogMap::Object(map) => Ok(IoSysMap::from((&*map).as_view())), - CrashLogMap::FirmwareData(log) => log.buf(), - } + Ok(IoSysMap::from((&self.map).as_view())) } } @@ -309,8 +225,8 @@ impl rtkit::Buffer for CrashLog { struct RoleState { role: Role, shared: Arc, - /// The fixed crash log buffer and its mapping; `None` where the firmware places its own. - crash_log: Option<(FixedBuffer, shmem::VMapOwned)>, + crash_log: FixedBuffer, + crash_map: shmem::VMapOwned, crashed: AtomicBool, #[pin] ack: Opaque, @@ -375,7 +291,7 @@ impl rtkit::Operations for RoleOps { if !is_events_message(ep, msg) { return false; } - data.shared.queue_event_records(); + data.shared.queue_events(); true } @@ -389,81 +305,26 @@ impl rtkit::Operations for RoleOps { iova: usize, size: usize, ) -> Result { - if let Some((buffer, map)) = data.crash_log.as_ref() { - if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != buffer.va || size > buffer.size { - return Err(EINVAL); - } - return Ok(CrashLog { - iova, - map: CrashLogMap::Object(map.owner().owned_vmap()?), - }); + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size + { + return Err(EINVAL); } - // The firmware places its buffers in its data segment, at their physical address. Ok(CrashLog { iova, - map: CrashLogMap::FirmwareData(data_segment::DataSegmentLog::map( - &data.shared.dev, - iova, - size, - )?), + map: data.crash_map.owner().owned_vmap()?, }) } } -/// The RTKit of one coprocessor. The mailbox driver serializes its own register writes; this -/// lock only serializes the exclusive borrows the RTKit API takes, so that publishers holding -/// different locks can ring the same doorbell. -#[pin_data] -struct Mailbox { - #[pin] - lock: Mutex<()>, - rtkit: core::cell::UnsafeCell>, -} - -// SAFETY: Every access to the RTKit goes through `with`, which holds the lock. -unsafe impl Sync for Mailbox {} - -impl Mailbox { - fn new(rtkit: rtkit::RtKit) -> Result>> { - KBox::pin_init( - pin_init!(Self { - lock <- new_mutex!((), "g17::Mailbox::lock"), - rtkit: core::cell::UnsafeCell::new(rtkit), - }), - GFP_KERNEL, - ) - } - - fn with(&self, f: impl FnOnce(Pin<&mut rtkit::RtKit>) -> R) -> R { - let _guard = self.lock.lock(); - // SAFETY: The lock excludes every other borrow of the RTKit. - f(Pin::new(unsafe { &mut *self.rtkit.get() })) - } - - fn send(&self, endpoint: u8, message: u64) -> Result { - self.with(|rtkit| rtkit.send_message(endpoint, message)) - } - - /// A handle on the receive thread's minimum utilization, usable from any context. - /// - /// # Safety - /// - /// The caller must not use the handle after this mailbox is dropped. - unsafe fn rx_util_min_handle(&self) -> rtkit::RxUtilMin { - // SAFETY: The caller's obligation is the handle's. - self.with(|rtkit| unsafe { rtkit.rx_util_min_handle() }) - } -} - /// A running firmware coprocessor. /// /// Dropping it stops the CPU before RTKit and the crash log buffer go away. struct Coprocessor { cpu: regs::CpuControl, running: bool, - rtkit: Pin>, + rtkit: rtkit::RtKit, state: Arc, - _crash_buffer: Option, + _crash_buffer: KernelObject, } impl Coprocessor { @@ -477,32 +338,27 @@ impl Coprocessor { role: Role, ) -> Result { let index = role as usize; - let (crash_buffer, crash_log) = match &cfg.crash_buffers { - CrashBuffers::Fixed(buffers) => { - let buffer = &buffers[index]; - let mut crash_buffer = KernelObject::new( - dev, - uat.kernel_lower_vm(), - Placement::At(buffer.va), - buffer.size, - mmu::PROT_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); - let crash_map = crash_buffer.kernel_map()?; - let fixed = FixedBuffer { - va: buffer.va, - size: buffer.size, - }; - (Some(crash_buffer), Some((fixed, crash_map))) - } - CrashBuffers::FirmwareData => (None, None), - }; + let buffer = &cfg.crash_buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + + let crash_map = crash_buffer.kernel_map()?; let state = Arc::pin_init( pin_init!(RoleState { role, shared: shared.clone(), - crash_log, + crash_log: FixedBuffer { + va: buffer.va, + size: buffer.size, + }, + crash_map, crashed: AtomicBool::new(false), ack <- Opaque::ffi_init(|ptr| { // SAFETY: ptr is valid uninitialized completion storage that is pinned @@ -516,7 +372,7 @@ impl Coprocessor { Ok(Coprocessor { cpu: regs::CpuControl::new(pdev, index)?, running: false, - rtkit: Mailbox::new(Self::receiver(dev, index, &state)?)?, + rtkit: Self::receiver(dev, index, &state)?, state, _crash_buffer: crash_buffer, }) @@ -545,30 +401,24 @@ impl Coprocessor { fn boot(&mut self) -> Result { self.cpu.start()?; self.running = true; - self.rtkit.with(|rtkit| rtkit.boot()) - } - - /// Starts the firmware endpoints and hands the firmware its initialization root. An - /// acknowledgment message is awaited here; one in status A is the caller's to poll. - fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { - self.rtkit.with(|mut rtkit| { - rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; - rtkit.as_mut().start_endpoint(EP_DOORBELL)?; - rtkit.send_message( - EP_FIRMWARE, - MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), - ) - })?; - match ack { - fw::variant::InitAck::Message => self.state.wait_for_ack(), - fw::variant::InitAck::Status => Ok(()), - } + Pin::new(&mut self.rtkit).boot() + } + + /// Starts the firmware endpoints and hands the firmware its initialization root. + fn init(&mut self, root_va: u64) -> Result { + let mut rtkit = Pin::new(&mut self.rtkit); + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + )?; + self.state.wait_for_ack() } /// Sends a host notification to the firmware. - fn notify(&self, msg: u64) -> Result { - trace::message(trace::MAILBOX, msg); - self.rtkit.send(EP_DOORBELL, msg) + fn notify(&mut self, msg: u64) -> Result { + Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) } /// Stops the CPU. The firmware keeps its state, but no longer runs. @@ -602,14 +452,6 @@ impl Bridge { Ok(bridge) } - /// Keeps the register block for a target that runs without the bridge. - fn unused(regs: regs::G17Resources) -> Bridge { - Bridge { - regs, - enabled: false, - } - } - fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -630,25 +472,15 @@ impl Drop for Bridge { /// dropped, the bridge is disabled once they are stopped, and the firmware objects and their /// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. struct Firmware { - /// The secondary instance, on GPUs whose firmware runs on two (A18 Pro). A single-instance - /// firmware (M4 Pro) carries both roles on the primary. - secondary: Option, + secondary: Coprocessor, primary: Coprocessor, bridge: Bridge, - /// Publication state shared with publishers that do not hold the device mutex. - port: Arc, - /// Fast-path submissions already accounted by the idle check. - seen_submissions: u64, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, - hold: power::Hold, registration_power_release_pending: bool, - variant: &'static fw::Variant, queues: runtime::Registry, pool: Arc, - /// Render kick ring storage, kept and reused for the life of the device. - render_rings: Arc, render_ids: Arc, init: initdata::InitData, _render_global: mmu::Vm, @@ -658,11 +490,8 @@ struct Firmware { impl Drop for Firmware { fn drop(&mut self) { - if let Some(secondary) = self.secondary.as_mut() { - secondary.stop(); - } + self.secondary.stop(); self.primary.stop(); - self.port.set_effort_full(false); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while // those references still keep the register block powered. @@ -671,15 +500,6 @@ impl Drop for Firmware { } impl Firmware { - /// The firmware instances this GPU runs. - fn roles(&self) -> &'static [Role] { - if self.secondary.is_some() { - &[Role::Primary, Role::Secondary] - } else { - &[Role::Primary] - } - } - /// Publishes controls only while the restart handshake is closed. The device mutex remains /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { @@ -699,115 +519,43 @@ impl Firmware { } fn release_registration_power(&mut self) -> Result { - // An idle hold keeps the cores powered past registration; it releases them itself. - if self.hold.active() { - self.registration_power_release_pending = false; - return Ok(()); - } power::release(self) } /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { - self.port.note_gpu_activity(); self.effort.submit(); self.apply_effort(power::FULL_EFFORT) } fn apply_effort(&mut self, effort: u32) -> Result { - Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) + Self::apply_shared_effort(&self.init, &mut self.effort, effort) } fn apply_shared_effort( init: &initdata::InitData, state: &mut power::Effort, - port: &port::Port, effort: u32, ) -> Result { if state.applied() == effort && init.effort()? == effort { return Ok(()); } init.publish_effort(effort)?; - trace::message(trace::EFFORT, u64::from(effort)); state.did_apply(effort); - port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) } /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, - /// unretired work; quarantined work retaining mappings is excluded. Submissions published - /// without the device mutex count as activity, and a clear that races one is abandoned. - fn observe_statistics_power(&mut self) { - if self.rings.statistics_suspended() { - return; - } - self.rings.observe_statistics_power( - self.bridge.regs.g17_core_power() & 0xf != 0, - self.port.statistics_epoch(), - ); - } - + /// unretired work; quarantined work retaining mappings is excluded. fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - let submissions = self.port.submissions(); - let submitted = submissions != self.seen_submissions; - self.seen_submissions = submissions; - if submitted { - self.effort.submit(); - } - let effort = match self.effort.idle(now_ns, in_flight || submitted) { - power::IdleAction::None => None, + match self.effort.idle(now_ns, in_flight) { + power::IdleAction::None => Ok(None), power::IdleAction::Clear => { - self.port.set_effort_full(false); - if self.port.submissions() != submissions { - self.port.set_effort_full(true); - self.effort.submit(); - return Ok(Some(power::IDLE_HOLD_NS)); - } self.apply_effort(0)?; - None - } - power::IdleAction::Recheck(delay) => Some(delay), - }; - Ok( - match (effort, self.idle_hold(now_ns, in_flight || submitted)) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - }, - ) - } - - /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A - /// refused control (recovery in progress) is retried on a later check. - fn idle_hold(&mut self, now_ns: u64, in_flight: bool) -> Option { - const RETRY_NS: u64 = 1_000_000; - let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); - for _ in 0..2 { - match self - .hold - .step(now_ns, in_flight, self.effort.submissions(), hold_ms) - { - power::HoldAction::None => return None, - power::HoldAction::Recheck(delay) => return Some(delay), - power::HoldAction::Assert => { - if self.set_idle_control(false).is_err() { - return None; - } - self.hold.asserted(); - } - power::HoldAction::Release => { - if self.set_idle_control(true).is_err() { - return Some(RETRY_NS); - } - self.hold.released(); - return None; - } + Ok(None) } + power::IdleAction::Recheck(delay) => Ok(Some(delay)), } - None - } - - fn set_idle_control(&mut self, enabled: bool) -> Result { - power::Control::set_idle(self, enabled) } } @@ -862,39 +610,14 @@ struct Shared { drm: Mutex>, clusters: u32, active_cores: u32, - variant: &'static fw::Variant, - /// Layout of compute queues' operand-state buffers for this GPU's cores. - operand_state: &'static fw::compute::OperandState, - /// Layout of renders' deflake buffers for this GPU's cores. - deflake: fw::render::Deflake, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, #[pin] feed_work: Work, preparations: Arc, - /// Compute admission waiting for a physical channel: signalled when a channel becomes - /// reusable or shareable, a QID or slot is returned, or a pool vacancy is witnessed. - channels: Arc, - /// Compute admission waiting for retirement-bound capacity (work states, ring and - /// in-flight room, drain witnesses, pool releases) or for publication to reopen. - progress: Arc, - /// Logical-queue creation waiting for a returned queue slot or execution context. - queues: Arc, - /// Render publication waiting for render capacity, a pair on loan, a work - /// state or firmware room: signalled when a pair, lease or work state - /// returns, an owner closes, a teardown parks, and on every event pass - /// while a render publisher waits (firmware consumption has no event). - renders: Arc, - /// Unbound compute publishers which can reuse a retired channel. - compute_demand: AtomicU32, #[pin] work: Work, - /// The next event pass runs every housekeeping step: set by every [`Self::queue_events`] - /// request, not by the firmware's event notification ([`Self::queue_event_records`]). - housekeeping: AtomicBool, - /// When an event pass last ran every housekeeping step, in ns. - housekeeping_ns: AtomicU64, #[pin] poll: DelayedWork, #[pin] @@ -915,8 +638,6 @@ struct Shared { state: Mutex>>, #[pin] changed: CondVar, - /// The port of the session in `state`; null before boot and once detachment begins. - port: core::sync::atomic::AtomicPtr, } impl_has_work! { @@ -1006,37 +727,8 @@ impl Shared { drop(retired); } - /// Wakes the waiters for a returned pair, binding or capacity change. - fn note_changed(&self) { - self.changed.notify_all(); - self.renders.notify(); - } - - /// The session's port, valid while `lease` is held: shutdown and device loss join every - /// lease before they detach the session. - fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { - let _ = lease; - // SAFETY: The pointer is published before the session becomes reachable and cleared - // only after every lease has been joined, so it names a live port for 'a. - unsafe { self.port.load(Ordering::Acquire).as_ref() } - } - - /// Called after joining every preparation lease, before the session is detached. - fn forget_port(&self) { - self.port.store(core::ptr::null_mut(), Ordering::Release); - } - - /// Queues the event worker for a pass that runs every housekeeping step. Callable from any - /// context. + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { - self.housekeeping.store(true, Ordering::Release); - self.queue_event_records(); - } - - /// Queues the event worker for firmware records alone: completions, or records that arrived - /// during a pass. The pass skips the housekeeping that nothing needs (see - /// `runtime_event_turn`). Callable from any context. - fn queue_event_records(self: &Arc) { // Already queued if this fails, which is just as good. let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); } @@ -1050,21 +742,12 @@ impl Shared { } fn drain_reclaims(self: &Arc) { - let mut reclaimed = false; loop { let next = self.reclaim.lock().take_one(); let Some(next) = next else { break; }; next.finish(); - reclaimed = true; - } - if reclaimed { - // Context release returns its execution root off-lock. The next - // device pass can now detach idle compute aliases and their QoS - // leases, without depending on a new client's submission/event. - self.queues.notify(); - self.queue_events(); } } @@ -1143,10 +826,6 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, - stats: Arc, - _capacity: Pin>>, - _reports: Pin>>, - _power: Pin>>, } impl Gpu { @@ -1157,22 +836,7 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { - let calibration = if cfg.chip_id == crate::hw::t6040::CHIP_ID { - Some(calibration::Calibration::read(dev).inspect_err(|_| { - dev_err!( - dev.as_ref(), - "Missing or invalid T6040 GPU calibration handoff\n" - ); - })?) - } else { - None - }; - let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; - if let Some(calibration) = calibration { - calibration.check_cores(&id.core_masks)?; - calibration.check_opps(&initdata::InitData::opps(dev, cfg.opps, id.num_clusters)?)?; - } if id.family != cfg.identity.family || id.variant != cfg.identity.variant || id.num_dies != cfg.identity.num_dies @@ -1192,74 +856,22 @@ impl Gpu { oas: cfg.uat_oas, }; let max_frequency_khz = Self::max_frequency_khz(dev)?; - let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let idle_delay = - power::idle_delay(node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?)?; - power::IDLE_HOLD_MS.store(idle_delay, Ordering::Relaxed); - dev_info!( - dev.as_ref(), - "GPU idle power-off delay: {} ms\n", - idle_delay - ); - let channels = availability::Availability::new()?; - let progress = availability::Availability::new()?; - let queues = availability::Availability::new()?; - let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; - let renders = availability::Availability::new()?; + let preparations = preparation::Gate::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); - let alternate = cfg.core_layout.cores.is_some_and(|(clusters, cores)| { - clusters == id.num_clusters - && cores != active_cores - && cfg.core_layout.alternate_core_masks.is_some_and(|(slots, enabled)| { - id.num_cores == slots - && id.core_masks.len() == clusters as usize - && id.core_masks.iter().all(|mask| mask.count_ones() == enabled) - }) - }); - if cfg - .core_layout - .cores - .is_some_and(|cores| cores != (id.num_clusters, active_cores)) - && !alternate - { - dev_err!( - dev.as_ref(), - "Unsupported GPU core layout: {} clusters with {} cores\n", - id.num_clusters, - active_cores - ); - return Err(ENODEV); - } - if alternate { - dev_warn!( - dev.as_ref(), - "Experimental GPU topology admission: command-buffer offsets are not yet validated\n" - ); - } let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, active_cores, - variant, - operand_state: &cfg.core_layout.operand_state, - deflake: cfg.core_layout.deflake, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, - channels, - progress, - queues, - renders, - compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), - housekeeping: AtomicBool::new(true), - housekeeping_ns: AtomicU64::new(0), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), grow <- new_work!("g17::Shared::grow"), @@ -1268,31 +880,10 @@ impl Gpu { stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), - port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), GFP_KERNEL, )?; - let directory = kernel::debugfs::Dir::new(c"aurora-g17"); - let capacity = KBox::pin_init( - directory.read_only_file(c"capacity", capacity::View(shared.clone())), - GFP_KERNEL, - )?; - let reports = KBox::pin_init( - directory.read_only_file(c"firmware_reports", reports::View(shared.clone())), - GFP_KERNEL, - )?; - let power = KBox::pin_init( - directory.read_only_file(c"power", reports::PowerView(shared.clone())), - GFP_KERNEL, - )?; - let stats = Arc::new( - crate::stats::StatsSnapshot { - g17_reader: Some(stats::Reader(shared.clone())), - ..Default::default() - }, - GFP_KERNEL, - )?; let gpu = Arc::new( Gpu { cfg, @@ -1301,80 +892,15 @@ impl Gpu { geometry, ids: Default::default(), shared, - stats, - _capacity: capacity, - _reports: reports, - _power: power, }, GFP_KERNEL, )?; let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; - gpu.shared - .port - .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); *gpu.shared.state.lock() = Some(firmware); gpu.shared.queue_events(); Ok(gpu) } - /// Returns the variant of the loaded firmware build. An identified SoC refuses a build it - /// does not know, before anything is written to the GPU. - fn firmware_variant( - dev: &AsahiDrmDevice, - firmware: &fw::variant::Firmware, - ) -> Result<&'static fw::Variant> { - let builds = match firmware { - fw::variant::Firmware::Fixed(variant) => return Ok(variant), - fw::variant::Firmware::Identified(builds) => builds, - }; - let version = Self::os_firmware_version(dev); - // iBoot strings can be shared by OS releases. Prefer the reported release's - // candidate, but admit only an exact loaded image, regardless of that hint. - let preferred = builds - .iter() - .filter(|build| Some(build.os_version) == version); - let others = builds - .iter() - .filter(|build| Some(build.os_version) != version); - for build in Iterator::chain(preferred, others) { - if Self::identify_image(dev, build).is_ok() { - dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); - return Ok(build.variant); - } - } - dev_err!( - dev.as_ref(), - "Unsupported GPU firmware image (reported macOS version {:?})\n", - version - ); - Err(ENODEV) - } - - /// Checks that the loaded TEXT (the `fw-text` region) is one of `build`'s images, with the DATA size - /// of the `fw-data` region. The version property is only a selection hint. - fn identify_image(dev: &AsahiDrmDevice, build: &'static fw::variant::Build) -> Result { - let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; - let text = node.reserved_mem_region_to_resource_byname(c_str!("fw-text"))?; - let data = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; - let start = text.start(); - let end = start.checked_add(text.size()).ok_or(EINVAL)?; - crate::fwid::identify(dev.as_ref().as_ref(), start..end, data.size(), build.images) - .inspect_err(|e| { - dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e) - })?; - Ok(()) - } - - /// The macOS version the boot loader reports for the boot firmware that loaded the GPU - /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. - fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { - let node = dev.as_ref().as_ref().of_node()?; - let cells = node - .get_property::>(c_str!("apple,firmware-version")) - .ok()?; - cells.as_slice().try_into().ok() - } - /// Returns the highest advertised operating-point frequency without changing table order. fn max_frequency_khz(dev: &AsahiDrmDevice) -> Result { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; @@ -1406,7 +932,6 @@ impl Gpu { mmu::UatFirmware::PublishedRoots { secondary_root_offset: cfg.secondary_root_offset, }, - cfg.uat_quirks, )?, GFP_KERNEL, )?; @@ -1421,30 +946,13 @@ impl Gpu { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let primary = - Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary).inspect_err(|e| { - dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e) - })?; - let secondary = match cfg.secondary_root_offset { - Some(_) => Some( - Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err( - |e| { - dev_err!( - dev.as_ref(), - "Secondary coprocessor setup failed: {:?}\n", - e - ) - }, - )?, - ), - None => None, - }; + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; let render_global = uat.new_vm(0, geometry.user_range())?; - let bridge = match cfg.fabric_bridge { - true => Bridge::enable(regs)?, - false => Bridge::unused(regs), - }; + let bridge = Bridge::enable(regs)?; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. @@ -1453,27 +961,12 @@ impl Gpu { secondary, primary, bridge, - port: port::Port::new( - power::Demand::new(pdev.as_ref().as_raw())?, - shared.feed.clone(), - )?, - seen_submissions: 0, - rings: channel::Rings::new(shared.variant.graph)?, + rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), - hold: power::Hold::new(), registration_power_release_pending: false, - variant: shared.variant, - queues <- runtime::Registry::with_waiters( - shared.clusters, - shared.deflake, - shared.descriptor_flags, - shared.variant, - Some(shared.queues.clone()), - Some(shared.renders.clone()), - ), + queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, - render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, @@ -1484,107 +977,45 @@ impl Gpu { )?; let session = &mut *firmware; session.primary.boot()?; - if let Some(secondary) = session.secondary.as_mut() { - secondary.boot()?; - } - // The firmware built its root and kernel-window tables while booting. - session.uat.invalidate_reserved_tables(); - let root_entries = match cfg.adopt_root_entries { - true => Some(session.uat.adopt_firmware_root_entries()?), - false => None, - }; + session.secondary.boot()?; + let root_entries = session.uat.adopt_firmware_root_entries()?; session.init.build( dev, cfg, &session.uat, &session._render_global, &session._pm_metrics, - shared.clusters, - shared.active_cores, - shared.variant, - )?; - if let Some(root_entries) = root_entries { - session.uat.confirm_firmware_root_entries(root_entries)?; - } - let mailbox: *const Mailbox = &*session.primary.rtkit; - session.port.attach( - session.init.port_views(mailbox)?, - session.bridge.regs.clone(), )?; + session.uat.confirm_firmware_root_entries(root_entries)?; session.uat.mirror_secondary_root()?; - if cfg.fabric_bridge { - session.bridge.regs.enable_g17_gating()?; - } - let ack = shared.variant.init_ack; + session.bridge.regs.enable_g17_gating()?; + session.primary.init(session.init.root_va(Role::Primary)?)?; session - .primary - .init(session.init.root_va(Role::Primary)?, ack)?; - if ack == fw::variant::InitAck::Status { - Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; - } - if let Some(secondary) = session.secondary.as_mut() { - secondary.init(session.init.root_va(Role::Secondary)?, ack)?; - } - if shared.variant.power_on_done { - session.primary.notify(MSG_POWER_ON_DONE)?; - if let Some(secondary) = session.secondary.as_mut() { - secondary.notify(MSG_POWER_ON_DONE)?; - } - } - Self::wait_for_opening( - dev, - &session.init, - &session.primary, - session.secondary.as_ref(), - )?; + .secondary + .init(session.init.root_va(Role::Secondary)?)?; + session.primary.notify(MSG_POWER_ON_DONE)?; + session.secondary.notify(MSG_POWER_ON_DONE)?; + Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; session .uat .activate_render_context(&session._render_global)?; Ok(firmware) } - /// Waits for a firmware that sends no acknowledgment message to accept its - /// initialization in status A. - fn wait_for_init_accepted( - dev: &AsahiDrmDevice, - init: &initdata::InitData, - primary: &Coprocessor, - ) -> Result { - for poll in 0..=INIT_ACCEPT_POLLS { - if primary.state.crashed.load(Ordering::Acquire) { - return Err(EIO); - } - if init.init_accepted()? { - return Ok(()); - } - if poll < INIT_ACCEPT_POLLS { - fsleep(Delta::from_millis(1)); - } - } - dev_err!( - dev.as_ref(), - "Firmware did not accept its initialization: status A {:#x?}\n", - init.status_read_words::<4>(Role::Primary, 0x10)? - ); - Err(ETIMEDOUT) - } - /// Waits for both coprocessors to retire their opening device-control records. fn wait_for_opening( dev: &AsahiDrmDevice, init: &initdata::InitData, primary: &Coprocessor, - secondary: Option<&Coprocessor>, + secondary: &Coprocessor, ) -> Result { for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) - || secondary.is_some_and(|s| s.state.crashed.load(Ordering::Acquire)) + || secondary.state.crashed.load(Ordering::Acquire) { return Err(EIO); } - if init.opening_retired(Role::Primary)? - && (secondary.is_none() || init.opening_retired(Role::Secondary)?) - { + if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { return Ok(()); } if poll < OPENING_RETIRE_POLLS { @@ -1629,7 +1060,7 @@ impl gpu::Gpu for Gpu { } fn stats_snapshot(&self) -> Option> { - Some(self.stats.clone()) + None } fn manager(&self) -> Option<&dyn gpu::GpuManager> { @@ -1650,20 +1081,12 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= - u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD) - | sync_output::FEATURE_CUMULATIVE_SIGNALS; - let variant = self.shared.variant; - if variant.few_primitives { - params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); - } - if variant.ray_traversal { - params.features |= - u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); - } - params.gpu_generation = self.cfg.gpu_generation as u32; + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY as u64; + params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; params.chip_id = self.cfg.chip_id; @@ -1692,18 +1115,9 @@ impl gpu::Gpu for Gpu { true } - /// The G17 runtime never programs fault_control into the firmware, so unmapped reads are not - /// guaranteed to return zero. Advertising soft faults made Mesa point out-of-bounds vec3/vec4 - /// vertex fetches at address 0 and skip bounds checks on bounded loads; on T6040 those reads - /// returned non-zero data (2026-10-19: every robustness2 rgba32 vertex_attribute_fetch case - /// failed; all pass with soft faults off). - fn supports_soft_faults(&self) -> bool { - false - } - fn queue_limits(&self) -> Option { Some(uapi::drm_asahi_queue_limits { - max_queues: kick::queue_limit(self.shared.variant.qid_floor), + max_queues: mmu::MAX_EXECUTION_CONTEXTS, max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, ..Default::default() }) @@ -1739,39 +1153,15 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); - let frontend = queue::Frontend::new( - priority, - usc_exec_base, - self.geometry.user_usable_range(), - self.shared.variant, - )?; + let frontend = + queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; let policy = scheduling_policy(priority)?; - let admission_started = now_ns(); let (backend, slot) = loop { - let observed = self.shared.queues.observe(); match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, - Err(error) if error == ENOSPC => { - // Closed logical queues can still own execution/QoS IDs - // until the scheduler release is consumed and reclaimed. - // Backend construction has rolled back all temporary leases. - let waited = now_ns().saturating_sub(admission_started); - let limit = u64::from(job::TIMEOUT_MS) * 1_000_000; - if waited >= limit { - return Err(EBUSY); - } - self.shared.queue_events(); - // A returned queue slot or reclaimed context wakes this. - self.shared.queues.block( - observed, - (limit - waited).div_ceil(1_000_000) as u32, - true, - )?; - } result => break result?, } }; @@ -1781,7 +1171,6 @@ impl gpu::Gpu for Gpu { backend.clone(), frontend, slot, - usage, ) { Ok(frontend) => frontend, Err(error) => { @@ -1796,32 +1185,27 @@ impl gpu::Gpu for Gpu { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - let mapping = { - let state = self.shared.state.lock(); - let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - bo.map_range_into_range( - firmware.uat.kernel_vm(), - range, - addresses, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - )? - }; - // The buffer's CPU mapping takes the reservation lock: not under the device mutex. - crate::timestamp::Buffer::firmware(mapping) + ) -> Result { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) } } /// Scheduling profile of a new logical queue. `file.rs` passes -/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses REALTIME, so HIGH -/// arrives as 1, MEDIUM as 2 and LOW as 3. +/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses HIGH and REALTIME, so +/// MEDIUM arrives as 2 and LOW as 3. fn scheduling_policy(priority: u32) -> Result { match priority { - 1 => Ok(hw::t8140::scheduling::HIGH), 2 => Ok(hw::t8140::scheduling::MEDIUM), 3 => Ok(hw::t8140::scheduling::LOW), _ => Err(EINVAL), diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 87fdb0e8a62ad..46f4e0c56b6ae 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -78,17 +78,7 @@ impl KernelObject { let mut this = Self::backing(dev, size, cpu)?; let mapping = match placement { Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, - Placement::In(range, align) => { - let site = if range.start >= crate::hw::t8140::dynamic::LOWER.start - && range.end <= crate::hw::t8140::dynamic::LOWER.end { - super::capacity::Site::LowerVa - } else { - super::capacity::Site::UpperVa - }; - this.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { - if error == ENOSPC { super::capacity::refused(site) } else { error } - })? - } + Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, }; this.mapping = Some(mapping); Ok(this) @@ -123,9 +113,6 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) - .map_err(|error| if error == ENOSPC { - super::capacity::refused(super::capacity::Site::AliasVa) - } else { error }) } /// Reuses only a complete alias of this exact retained backing. Permanent @@ -172,20 +159,6 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) - .map_err(|error| if error == ENOSPC { - super::capacity::refused(super::capacity::Site::AliasVa) - } else { error }) - } - - fn alias_in_lowest( - &self, - vm: &mmu::Vm, - range: Range, - align: u64, - prot: mmu::Prot, - guard_size: usize, - ) -> Result { - vm.map_in_range_lowest(&self.obj.gem, 0..self.size, align, range, prot, guard_size) } /// Maps an exact backing subrange, retaining an explicit unmapped guard. @@ -368,76 +341,6 @@ impl Allocator<'_> { cpu, ) } - - /// Place fresh backing at one address in the canonical and both alias VMs. - /// A client may retain mappings after their canonical reservation is freed. - /// Retry only address collisions, keeping the same backing throughout. - pub(crate) fn lower_with_aliases( - &self, - size: usize, - align: u64, - prot: mmu::Prot, - cpu: CpuMap, - aliases: [&mmu::Vm; 2], - ) -> Result<(KernelObject, [mmu::KernelMapping; 2])> { - if size == 0 || size % mmu::UAT_PGSZ != 0 { - return Err(EINVAL); - } - let mut object = KernelObject::backing(self.dev, size, cpu)?; - let mut range = crate::hw::t8140::dynamic::LOWER; - let mut searching = false; - loop { - // Keep ordinary best-fit placement on the uncontended path. A - // collision starts a lowest-address search over the whole arena. - let mapping = (if searching { - object.alias_in_lowest( - self.uat.kernel_lower_vm(), range.clone(), align, prot, mmu::UAT_PGSZ, - ) - } else { - object.obj.map_into_range( - self.uat.kernel_lower_vm(), range.clone(), align, prot, true, - ) - }).map_err(|error| if error == ENOSPC { - crate::g17::capacity::refused(crate::g17::capacity::Site::LowerVa) - } else { error })?; - let address = mapping.iova(); - let end = address.checked_add(size as u64).ok_or(EOVERFLOW)?; - // Partial aliases are destroyed before releasing the canonical - // reservation, so a retry cannot expose its address prematurely. - let mapped: core::result::Result<_, (Error, usize)> = (|| { - let first = object.map_alias(aliases[0], address, prot) - .map_err(|error| (error, 0))?; - let second = object.map_alias(aliases[1], address, prot) - .map_err(|error| (error, 1))?; - Ok([first, second]) - })(); - match mapped { - Ok(mapped) => { - object.mapping = Some(mapping); - return Ok((object, mapped)); - } - Err((error, index)) if error == ENOSPC && end < range.end => { - drop(mapping); - let start = if searching { - address + mmu::UAT_PGSZ as u64 - } else { - range.start - }; - // Jump to an actual hole in the VM that refused the alias, - // rather than remapping backing at every intervening page. - let candidate = object.alias_in_lowest( - aliases[index], start..range.end, align, prot, 0, - ).map_err(|error| if error == ENOSPC { - crate::g17::capacity::refused(crate::g17::capacity::Site::AliasVa) - } else { error })?; - range.start = candidate.iova(); - drop(candidate); - searching = true; - } - Err((error, _)) => return Err(error), - } - } - } } /// Families whose upper mappings remain cached and installed until firmware @@ -466,17 +369,8 @@ struct PoolInner { reserved: [usize; POOL_FAMILIES], bytes: usize, stopped: bool, - /// Objects the firmware may still reference after their owner dropped them, kept mapped - /// and out of reuse until firmware stop (see [`PooledObject::retain_until_stop`]). - kept: KVec, - kept_bytes: usize, - /// Kept objects still owned by their users, whose list entries are reserved. - kept_reserved: usize, } -/// Retained objects past this many are still retained, with one warning. -const KEPT_WARN: usize = 256; - /// Per-device cache-safe object recycling. Every converted mapping reserves /// one return slot before its page-table attributes change. Active leases and /// poisoned mappings count against the same capacity as reusable objects. @@ -499,9 +393,6 @@ struct CacheLease { pub(crate) struct PooledObject { object: core::mem::ManuallyDrop, lease: Option, - /// The pool keeping this object until firmware stop, and whether `reserve_kept` took its - /// list entry. - keep: Option<(kernel::sync::Arc, bool)>, } impl core::ops::Deref for PooledObject { @@ -515,28 +406,11 @@ impl core::ops::DerefMut for PooledObject { &mut self.object } } -impl PooledObject { - /// The firmware may still reference this object after its owner drops it: keep it mapped - /// and never hand it out again before firmware stop. It leaves the reuse cache, so it - /// does not take cache capacity. Call from sleepable context: this reserves the list - /// entry, so the drop, wherever it happens, does not allocate. - pub(crate) fn retain_until_stop(&mut self, pool: &kernel::sync::Arc) { - if self.keep.is_none() { - let reserved = pool.reserve_kept(); - self.keep = Some((pool.clone(), reserved)); - } - } -} - impl Drop for PooledObject { fn drop(&mut self) { // SAFETY: This destructor is the only operation that moves `object` // out; ManuallyDrop prevents a second implicit drop of the field. let object = unsafe { core::mem::ManuallyDrop::take(&mut self.object) }; - if let Some((pool, reserved)) = self.keep.take() { - pool.keep_object(object, self.lease.take(), reserved); - return; - } match self.lease.take() { Some(lease) => lease.pool.return_object(object, &lease), None => drop(object), @@ -552,10 +426,7 @@ impl Pool { } kernel::sync::Arc::pin_init( pin_init!(Self { - inner <- kernel::new_mutex!(PoolInner { - returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false, kept: KVec::new(), - kept_bytes: 0, kept_reserved: 0, - }, "g17::object::Pool"), + inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false }, "g17::object::Pool"), }), GFP_KERNEL, ) @@ -604,7 +475,6 @@ impl Pool { return Ok(PooledObject { object: core::mem::ManuallyDrop::new(entry.object), lease: Some(lease()), - keep: None, }); } let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?; @@ -629,7 +499,6 @@ impl Pool { let mut result = PooledObject { object: core::mem::ManuallyDrop::new(object), lease: reserved.then(lease), - keep: None, }; if !reserved { return Ok(result); @@ -689,67 +558,10 @@ impl Pool { core::mem::forget(object); } - /// Room for one more kept object, taken by the matching `keep_object`. Returns whether - /// it was reserved; after a failed allocation that drop leaks the object instead. - fn reserve_kept(&self) -> bool { - let mut inner = self.inner.lock(); - let pending = inner.kept_reserved + 1; - if inner.kept.capacity() - inner.kept.len() < pending - && inner.kept.reserve(pending, GFP_KERNEL).is_err() - { - return false; - } - inner.kept_reserved = pending; - true - } - - fn keep_object(&self, object: KernelObject, lease: Option, reserved: bool) { - let mut inner = self.inner.lock(); - if reserved { - inner.kept_reserved = inner.kept_reserved.saturating_sub(1); - } - if let Some(lease) = &lease { - // The object leaves the reuse cache and its capacity. - inner.reserved[lease.family as usize] -= 1; - inner.bytes -= object.size(); - } - if inner.stopped { - drop(inner); - drop(object); - return; - } - let size = object.size(); - if inner.kept.len() == KEPT_WARN { - pr_warn!("G17 keeps {} firmware-linked objects until the GPU firmware restarts\n", KEPT_WARN); - } - if !reserved { - // Another kept object's reservation must not be used: leak this one instead. - pr_err!("G17 could not record a firmware-linked object; leaking it\n"); - core::mem::forget(object); - return; - } - // retain_until_stop reserved the entry; this never allocates. - match inner.kept.push_within_capacity(object) { - Ok(()) => inner.kept_bytes += size, - Err(error) => { - // Never unmap an object the firmware may still reference. - pr_err!("G17 could not record a firmware-linked object; leaking it\n"); - core::mem::forget(error.0); - } - } - } - /// Releases retained mappings after both firmware processors have stopped. /// Active objects returned later observe the same stop witness. pub(crate) fn stop_after_firmware(&self) { - let kept = { - let mut inner = self.inner.lock(); - inner.stopped = true; - inner.kept_bytes = 0; - core::mem::take(&mut inner.kept) - }; - // Mapping teardown must not run while the pool mutex is held. - drop(kept); + self.inner.lock().stopped = true; for index in 0..POOL_FAMILIES * POOL_SLOTS { let entry = { let mut inner = self.inner.lock(); @@ -765,17 +577,3 @@ impl Pool { } } } - -impl Pool { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - let inner = self.inner.lock(); - for family in 0..POOL_FAMILIES { - let entries = &inner.returned[family * POOL_SLOTS..(family + 1) * POOL_SLOTS]; - writeln!(f, "cached_objects_{} used={} total={} returned={} poisoned={}", family, - inner.reserved[family], POOL_SLOTS, entries.iter().flatten().count(), - entries.iter().flatten().filter(|entry| entry.poisoned).count())?; - } - writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES)?; - writeln!(f, "firmware_linked_objects kept={} bytes={}", inner.kept.len(), inner.kept_bytes) - } -} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index a2e293df91510..96b363bc27447 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -3,30 +3,8 @@ //! Work-based firmware performance effort. Idle power gating is managed separately by the //! device-control ring; this input never keeps an otherwise idle GPU powered. -use core::sync::atomic::AtomicU32; - pub(super) const FULL_EFFORT: u32 = 0x10000; -pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; - -/// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after -/// the last job. Not measured on other firmware; at or below it the hold stays off. -pub(crate) const FIRMWARE_IDLE_OFF_MS: u32 = 2; - -/// Idle policy initialized from the existing DT `apple,idle-off-delay-ms` property. -/// The existing `idle_off_delay_ms` sysfs attribute may override it. Above -/// [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU cores powered until it has been idle -/// this long ([`Hold`]). -pub(crate) static IDLE_HOLD_MS: AtomicU32 = AtomicU32::new(FIRMWARE_IDLE_OFF_MS); - -/// Use the same bounds as the existing idle-delay attribute. Missing DT policy -/// preserves the native delay; malformed or out-of-range values fail probe. -pub(super) fn idle_delay(requested: Option) -> kernel::error::Result { - let delay = requested.unwrap_or(FIRMWARE_IDLE_OFF_MS); - if !(FIRMWARE_IDLE_OFF_MS..=1000).contains(&delay) { - return Err(kernel::error::code::EINVAL); - } - Ok(delay) -} +const IDLE_HOLD_NS: u64 = 16_000_000; #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum IdleAction { @@ -39,7 +17,6 @@ pub(super) struct Effort { streaming: bool, idle_since: Option, applied: u32, - submissions: u64, } impl Effort { @@ -48,19 +25,12 @@ impl Effort { streaming: false, idle_since: None, applied: 0, - submissions: 0, } } pub(super) fn submit(&mut self) { self.streaming = true; self.idle_since = None; - self.submissions = self.submissions.wrapping_add(1); - } - - /// Count of published submissions, for [`Hold::step`]. - pub(super) fn submissions(&self) -> u64 { - self.submissions } pub(super) fn applied(&self) -> u32 { @@ -92,83 +62,6 @@ impl Effort { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(super) enum HoldAction { - None, - /// Send the keep-powered idle policy. - Assert, - /// Send the power-off idle policy again. - Release, - Recheck(u64), -} - -/// Host-side idle hold. On T6040/25G83 the firmware powers the cores off 2 ms after the last job -/// and reads that delay only at boot; every wake-up then costs about 1 ms. With a longer -/// [`IDLE_HOLD_MS`] the host sends the keep-powered idle policy once work appears and the power-off -/// policy once no work has been submitted or in flight for that long. Checked from the event -/// worker, which runs on every completion, well inside the firmware's 2 ms. -pub(super) struct Hold { - active: bool, - seen: u64, - since: Option, -} - -impl Hold { - pub(super) const fn new() -> Self { - Self { - active: false, - seen: 0, - since: None, - } - } - - pub(super) fn active(&self) -> bool { - self.active - } - - pub(super) fn step( - &mut self, - now_ns: u64, - in_flight: bool, - submissions: u64, - hold_ms: u32, - ) -> HoldAction { - let enabled = hold_ms > FIRMWARE_IDLE_OFF_MS; - let fresh = submissions != self.seen; - self.seen = submissions; - if in_flight || fresh { - self.since = None; - } - if !self.active { - return if enabled && (in_flight || fresh) { - HoldAction::Assert - } else { - HoldAction::None - }; - } - if in_flight { - return HoldAction::None; - } - let hold_ns = if enabled { u64::from(hold_ms) * 1_000_000 } else { 0 }; - let since = *self.since.get_or_insert(now_ns.max(1)); - let idle = now_ns.saturating_sub(since); - if idle >= hold_ns { - HoldAction::Release - } else { - HoldAction::Recheck(hold_ns - idle) - } - } - - pub(super) fn asserted(&mut self) { - self.active = true; - } - - pub(super) fn released(&mut self) { - self.active = false; - self.since = None; - } -} - /// Single-producer control path under the device mutex. Event draining must not publish another /// control between `set_idle` and the exact three-counter retirement witness. pub(super) trait Control { @@ -244,99 +137,3 @@ pub(super) fn release(control: &mut C) -> kernel::error::Result { } result } - -#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_idle_hold))] -mod tests { - use super::*; - - #[test] - fn dt_idle_delay_is_bounded_and_absence_preserves_native_policy() { - assert_eq!(idle_delay(None), Ok(FIRMWARE_IDLE_OFF_MS)); - for delay in [2, 16, 1000] { - assert_eq!(idle_delay(Some(delay)), Ok(delay)); - } - for delay in [0, 1, 1001, u32::MAX] { - assert!(idle_delay(Some(delay)).is_err()); - } - } - - #[test] - fn frame_hold_survives_depth_one_gaps_but_releases_at_idle() { - let mut hold = Hold::new(); - assert_eq!(hold.step(0, true, 1, 16), HoldAction::Assert); - hold.asserted(); - for submission in 1..=8 { - // A completed depth-one dispatch followed by another short gap. - let now = submission * 3_000_000; - assert_eq!(hold.step(now, false, submission, 16), HoldAction::Recheck(16_000_000)); - assert!(hold.active()); - } - assert_eq!(hold.step(39_999_999, false, 8, 16), HoldAction::Recheck(1)); - assert_eq!(hold.step(40_000_000, false, 8, 16), HoldAction::Release); - hold.released(); - assert!(!hold.active()); - } - - #[test] - fn holds_until_idle_for_the_delay_then_releases() { - let mut hold = Hold::new(); - assert_eq!(hold.step(0, true, 1, 10), HoldAction::Assert); - hold.asserted(); - assert_eq!(hold.step(1_000_000, false, 1, 10), HoldAction::Recheck(10_000_000)); - assert_eq!(hold.step(6_000_000, false, 1, 10), HoldAction::Recheck(5_000_000)); - // A new submission restarts the idle period. - assert_eq!(hold.step(7_000_000, false, 2, 10), HoldAction::Recheck(10_000_000)); - assert_eq!(hold.step(17_000_000, false, 2, 10), HoldAction::Release); - hold.released(); - assert!(!hold.active()); - } - - #[test] - fn the_default_delay_never_holds_and_lowering_it_releases() { - let mut hold = Hold::new(); - assert_eq!(hold.step(0, true, 1, FIRMWARE_IDLE_OFF_MS), HoldAction::None); - assert_eq!(hold.step(0, true, 2, 10), HoldAction::Assert); - hold.asserted(); - assert_eq!(hold.step(1, false, 2, FIRMWARE_IDLE_OFF_MS), HoldAction::Release); - } -} - -unsafe extern "C" { - fn asahi_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int; - fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); - fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void); -} - -/// DT-selected memory-demand provider. Its independent allocation outlives a -/// detached provider; a device link orders supplier removal and suspend. -pub(super) struct Demand(*mut core::ffi::c_void); - -// SAFETY: the provider's leaf spinlock serializes all access; no GPU callback. -unsafe impl Send for Demand {} -// SAFETY: shared access only changes that locked request, never its lifetime. -unsafe impl Sync for Demand {} - -impl Demand { - pub(super) fn new(dev: *mut kernel::bindings::device) -> kernel::error::Result { - let mut out = core::ptr::null_mut(); - // SAFETY: live platform device and writable result; C returns an owned reference. - let ret = unsafe { asahi_clpc_gpu_get(dev.cast(), &mut out) }; - if ret < 0 { - Err(kernel::error::Error::from_errno(ret)) - } else { - Ok(Self(out)) - } - } - - pub(super) fn set_active(&self, active: bool) { - // SAFETY: the retained reference is live; NULL denotes no DT provider. - unsafe { asahi_clpc_gpu_set_active(self.0, active) }; - } -} - -impl Drop for Demand { - fn drop(&mut self) { - // SAFETY: releases exactly the reference returned by get, clearing demand first. - unsafe { asahi_clpc_gpu_put(self.0) }; - } -} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index c7d1761699a24..8563beda979e6 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -6,7 +6,6 @@ //! the device mutex, then drops that mutex before waiting for builders. A builder's lease is //! stored after its tentative mappings so those mappings are destroyed before the join completes. -use core::cell::Cell; use core::sync::atomic::{AtomicU64, Ordering}; use kernel::{ new_condvar, new_mutex, @@ -21,46 +20,6 @@ const REMOVED: u64 = CLOSED << 1; const FAILED: u64 = CLOSED << 2; const EPOCH: u64 = CLOSED << 3; -/// One ready packet's monotonic compute-admission budget. The callback checks -/// packet cancellation, logical close, VM status and device loss even when no -/// capacity/recovery event arrives. Wakes and nested constructors never renew it. -pub(super) struct Admission<'a> { - deadline: Cell, - started: u64, - check: &'a dyn Fn() -> Result, -} - -impl<'a> Admission<'a> { - pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { - Self { deadline: Cell::new(deadline), started: super::now_ns(), check } - } - - /// A recovery is pending or this owner's spared work drains: wait as long as the - /// watchdog may take to settle it (RECOVERY_WAIT_MS from the start) instead of failing. - pub(super) fn extend_for_recovery(&self) { - let limit = self - .started - .saturating_add(u64::from(super::timeout::RECOVERY_WAIT_MS) * 1_000_000); - if self.deadline.get() < limit { - self.deadline.set(limit); - } - } - - pub(super) fn check(&self) -> Result { - (self.check)()?; - if super::now_ns() >= self.deadline.get() { return Err(EBUSY); } - Ok(()) - } - - /// Ignore condvar remaining-time reports: only the monotonic deadline counts. - pub(super) fn slice_ms(&self) -> Result { - self.check()?; - let remaining = self.deadline.get().saturating_sub(super::now_ns()); - if remaining == 0 { return Err(EBUSY); } - Ok(remaining.div_ceil(1_000_000).min(10) as u32) - } -} - #[pin_data] pub(super) struct Gate { state: AtomicU64, @@ -68,37 +27,20 @@ pub(super) struct Gate { wait: Mutex<()>, #[pin] changed: CondVar, - /// Scheduler admission waiting for publication to reopen. - available: Option>, } impl Gate { - #[cfg(CONFIG_KUNIT = "y")] pub(super) fn new() -> Result> { - Self::with_waiters(None) - } - - /// A gate whose reopening, failure and removal also wake `available`. - pub(super) fn with_waiters( - available: Option>, - ) -> Result> { Arc::pin_init( pin_init!(Self { state: AtomicU64::new(EPOCH), wait <- new_mutex!((), "g17::Gate::wait"), changed <- new_condvar!("g17::Gate::changed"), - available, }), GFP_KERNEL, ) } - fn wake_waiters(&self) { - if let Some(available) = &self.available { - available.notify(); - } - } - pub(super) fn try_enter(self: &Arc) -> Result { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -150,28 +92,6 @@ impl Gate { } } - /// Only a recovery closes the gate: packet admission waits for it up to - /// RECOVERY_WAIT_MS from the admission's start (Admission::extend_for_recovery). - /// Called with no device mutex or existing preparation lease. - pub(super) fn enter_until(self: &Arc, admission: &Admission<'_>) -> Result { - admission.check()?; - match self.try_enter() { - Err(error) if error == EAGAIN => admission.extend_for_recovery(), - result => return result, - } - let mut wait = self.wait.lock(); - loop { - admission.check()?; - match self.try_enter() { - Err(error) if error == EAGAIN => admission.extend_for_recovery(), - result => return result, - } - let _ = self.changed.wait_timeout( - &mut wait, kernel::time::msecs_to_jiffies(admission.slice_ms()?) - ); - } - } - fn invalidate(&self, flags: u64) { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -193,66 +113,20 @@ impl Gate { } } - /// Also wakes publishers in wait_closure(); they then block in enter(). pub(super) fn close(&self) { self.invalidate(0); - let _wait = self.wait.lock(); - self.changed.notify_all(); - } - - /// Opaque snapshot for wait_closure(), taken where recovery was observed. - pub(super) fn snapshot(&self) -> u64 { - self.state.load(Ordering::Acquire) - } - - /// A publisher that observed recovery (host pending, or the firmware's state - /// ahead of the event worker, which always follows with a restart and the - /// closure) sleeps until this gate has closed since `seen`. enter() then - /// blocks until recovery reopens it. The deadline is only a failure bound. - pub(super) fn wait_closure(&self, seen: u64, deadline: u64) -> Result { - let mut wait = self.wait.lock(); - loop { - let state = self.state.load(Ordering::Acquire); - if state & (REMOVED | FAILED) != 0 { - return Err(ENODEV); - } - if state & CLOSED != 0 || state & !COUNT_MASK != seen & !COUNT_MASK { - return Ok(()); - } - let remaining = deadline.saturating_sub(super::now_ns()); - if remaining == 0 { - return Err(ETIMEDOUT); - } - let ms = u32::try_from(remaining.div_ceil(1_000_000)).unwrap_or(u32::MAX); - let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(ms)); - } } pub(super) fn remove(&self) { self.invalidate(REMOVED); - let wait = self.wait.lock(); + let _wait = self.wait.lock(); self.changed.notify_all(); - drop(wait); - self.wake_waiters(); } pub(super) fn fail(&self) { self.invalidate(FAILED); - let wait = self.wait.lock(); + let _wait = self.wait.lock(); self.changed.notify_all(); - drop(wait); - self.wake_waiters(); - } - - /// Runtime failure/recovery must not join an indefinitely stalled builder. - /// Timeout retains every graph and reservation; it never authorizes freeing. - pub(super) fn wait_drained_until(&self, deadline: u64) -> Result { - let mut wait = self.wait.lock(); - while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { - if super::now_ns() >= deadline { return Err(ETIMEDOUT); } - let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(10)); - } - Ok(()) } /// Join builders with no device, queue or mapping lock held. @@ -265,7 +139,7 @@ impl Gate { /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. pub(super) fn reopen(&self) -> bool { - let wait = self.wait.lock(); + let _wait = self.wait.lock(); let mut previous = self.state.load(Ordering::Acquire); loop { if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { @@ -279,8 +153,6 @@ impl Gate { ) { Ok(_) => { self.changed.notify_all(); - drop(wait); - self.wake_waiters(); return true; } Err(current) => previous = current, @@ -312,118 +184,3 @@ impl Drop for Lease { } } } - -#[kunit_tests(asahi_g17_growth_gate)] -mod growth_gate_tests { - use super::*; - #[test] - fn recovery_can_join_between_growth_admissions() { - let gate = Gate::new().unwrap(); - let outer = gate.try_enter().unwrap(); - // The growth handoff must drop this lease before enter can wait. - drop(outer); - gate.close(); - assert!(matches!(gate.try_enter(), Err(error) if error == EAGAIN)); - gate.wait_drained(); - assert!(gate.reopen()); - let growth = gate.enter(false).unwrap(); - assert!(growth.is_current()); - drop(growth); - } -} - -#[kunit_tests(asahi_g17_compute_deadline)] -mod compute_deadline_tests { - use super::*; - - #[test] - fn a_recovery_extends_the_deadline_once_from_the_start() { - let check = || Ok(()); - let admission = Admission::new(super::super::now_ns(), &check); - assert_eq!(admission.check(), Err(EBUSY)); - admission.extend_for_recovery(); - assert_eq!(admission.check(), Ok(())); - let limit = admission.started - + u64::from(super::super::timeout::RECOVERY_WAIT_MS) * 1_000_000; - assert_eq!(admission.deadline.get(), limit); - admission.extend_for_recovery(); - assert_eq!(admission.deadline.get(), limit); - } - - #[test] - fn spent_budget_refuses_even_an_open_gate() { - let gate = Gate::new().unwrap(); - let check = || Ok(()); - let admission = Admission::new(super::super::now_ns(), &check); - assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); - assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); - } - - #[test] - fn closed_gate_waits_past_the_budget_for_recovery() { - let gate = Gate::new().unwrap(); - gate.close(); - let start = super::super::now_ns(); - // Cancel well after the 20 ms budget: only the recovery extension keeps it waiting. - let check = || { - if super::super::now_ns() - start >= 60_000_000 { Err(ECANCELED) } else { Ok(()) } - }; - let admission = Admission::new(start + 20_000_000, &check); - assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); - assert!(super::super::now_ns() - start < 500_000_000); - gate.wait_drained(); - assert!(gate.reopen()); - } - - #[test] - fn cancellation_without_capacity_or_recovery_events() { - let gate = Gate::new().unwrap(); - gate.close(); - let start = super::super::now_ns(); - let check = || { - if super::super::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } - }; - let admission = Admission::new(start + 2_000_000_000, &check); - assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); - assert!(super::super::now_ns() - start < 500_000_000); - assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); - } - - #[test] - fn recovery_and_binding_retry_churn_cannot_renew_budget() { - let gate = Gate::new().unwrap(); - let deadline = super::super::now_ns() + 20_000_000; - let check = || Ok(()); - let admission = Admission::new(deadline, &check); - loop { - // Re-entry models a channel taken between admission/publication. - // Recovery joins each dropped lease, then wakes the same budget. - gate.close(); - gate.changed.notify_all(); - gate.wait_drained(); - assert!(gate.reopen()); - // However often the gate reopened, an attempt that starts at the deadline - // is refused. Timing the whole loop would also count time descheduled. - let attempted = super::super::now_ns(); - match gate.enter_until(&admission) { - Ok(lease) => { - assert!(attempted < deadline); - drop(lease); - } - Err(error) => { assert_eq!(error, EBUSY); break; } - } - } - assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); - } - - #[test] - fn packet_status_and_cancel_precede_available_capacity() { - let gate = Gate::new().unwrap(); - for error in [ECANCELED, EIO] { - let check = || Err(error); - let admission = Admission::new(u64::MAX, &check); - assert!(matches!(gate.enter_until(&admission), Err(got) if got == error)); - } - assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); - } -} diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index f09e629aa047e..1c0d6822c395f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -9,7 +9,6 @@ //! a graph or waiting for GPU work. Capacity waiters sleep on the release event. use crate::g17::{ - fw::kick::KickDependency, hazard::{self, Footprint, Plan}, job::Packet, }; @@ -44,46 +43,22 @@ struct InFlight { slot: Option, ticket: u64, footprint: Arc, - /// Fragment kick of the published render; a later same-VM overlap of - /// another pair waits for it in firmware. - fragment: Option, } const QUARANTINE_WAIT_MS: u32 = 1000; -/// A wake never renews admission's budget. Short observations also notice a -/// packet cancellation that has no corresponding pair-release notification. -fn wait_slice_ms(deadline_ns: u64, now_ns: u64) -> Result { - let remaining = deadline_ns.saturating_sub(now_ns); - if remaining == 0 { - return Err(EBUSY); - } - Ok(remaining.div_ceil(1_000_000).min(10) as u32) -} - struct State { active: u64, refs: [u8; cfg::RENDER_SLOTS], owners: [u64; cfg::RENDER_SLOTS], epochs: [u64; cfg::RENDER_SLOTS], quarantined: u64, - /// Quarantined slots whose failed work is all spared (innocent): waiting publications fail - /// their own fence with EBUSY instead of a device-loss error, so the VM stays usable. - spared: u64, kill_hold: u64, - /// Slots whose kill hold ended at their killed context's consumed scheduler release, - /// and when. A new owner takes such a pair last, the one released longest ago first. - killed: u64, - killed_at: [u64; cfg::RENDER_SLOTS], - /// The physical queue table still has room for a pair built in an unowned slot. The - /// runtime updates this whenever the table changes. - build_room: bool, released: u64, epoch: u64, footprints: KVec, next_ticket: u64, } - impl State { const MASK: u64 = (u64::MAX >> (64 - cfg::RENDER_SLOTS)) & !1; const fn new() -> Self { @@ -93,11 +68,7 @@ impl State { owners: [0; cfg::RENDER_SLOTS], epochs: [0; cfg::RENDER_SLOTS], quarantined: 0, - spared: 0, kill_hold: 0, - killed: 0, - killed_at: [0; cfg::RENDER_SLOTS], - build_room: true, released: 0, epoch: 0, footprints: KVec::new(), @@ -121,7 +92,7 @@ impl State { & Self::MASK == 0 => { - Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RenderSlots)) + Err(ENOSPC) } _ => Ok(()), } @@ -131,15 +102,6 @@ impl State { self.epochs[slot] = self.epoch; } fn take_idle(&mut self, slot: usize, owner: u64) -> u8 { - if self.owners[slot] != owner { - crate::g17::trace::teardown( - 6, - slot as u64, - self.owners[slot], - owner | u64::from(self.killed & (1u64 << slot) != 0) << 63, - ); - self.killed &= !(1u64 << slot); - } self.refs[slot] += 1; self.active |= 1u64 << slot; self.released &= !(1u64 << slot); @@ -153,20 +115,18 @@ impl State { } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. - let idle = (0..cfg::RENDER_SLOTS) - .filter(|slot| { - self.owners[*slot] == owner - && self.refs[*slot] == 0 - && self.quarantined & (1u64 << slot) == 0 - }) - .max_by_key(|slot| (self.epochs[*slot], usize::MAX - *slot)); - if let Some(slot) = idle { - if !(0..cfg::RENDER_SLOTS) - .any(|other| self.owners[other] == owner && self.refs[other] != 0) + for slot in 0..cfg::RENDER_SLOTS { + if self.owners[slot] == owner + && self.refs[slot] == 0 + && self.quarantined & (1u64 << slot) == 0 { - self.claim_sibling(owner); + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) + { + self.claim_sibling(owner); + } + return Some(self.take_idle(slot, owner)); } - return Some(self.take_idle(slot, owner)); } if self.owners.contains(&owner) { // Consecutive kicks are ordered by firmware dependencies. Stack @@ -185,23 +145,13 @@ impl State { } let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; let eligible = |slot: usize| free & (1u64 << slot) != 0; - let released = |slot: usize| self.released & (1u64 << slot) != 0; - let unowned = || (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0); - // Reuse a closed, exactly retired owner before consuming a new graph. A killed - // context's pair comes after a new graph while one can still be built: on Neo, new - // owners of such pairs had renders that executed but never stored their stamps. - // Once the queue table is full it is reused, the one released longest ago first, - // so killed pairs never strand physical queues. + // Reuse a closed, exactly retired owner before consuming a new graph. let slot = (0..cfg::RENDER_SLOTS) - .filter(|slot| eligible(*slot) && released(*slot) && self.killed & (1u64 << slot) == 0) + .filter(|slot| eligible(*slot) && self.released & (1u64 << slot) != 0) .min_by_key(|slot| self.epochs[*slot]) - .or_else(|| if self.build_room { unowned() } else { None }) .or_else(|| { - (0..cfg::RENDER_SLOTS) - .filter(|slot| eligible(*slot) && released(*slot)) - .min_by_key(|slot| self.killed_at[*slot]) - }) - .or_else(unowned)?; + (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) + })?; let first = self.take_idle(slot, owner); self.claim_sibling(owner); Some(first) @@ -210,8 +160,7 @@ impl State { if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { return; } - // A speculative sibling never takes a killed context's pair. - let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold & !self.killed; + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; let eligible = |slot: usize| free & (1u64 << slot) != 0; let sibling = (0..cfg::RENDER_SLOTS) .filter(|slot| { @@ -221,61 +170,13 @@ impl State { }) .min_by_key(|slot| self.epochs[*slot]) .or_else(|| { - // Nor an unowned slot once no pair can be built there. - if !self.build_room { - return None; - } (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) }); if let Some(slot) = sibling { self.released &= !(1u64 << slot); self.owners[slot] = owner; - // A speculative sibling has no GPU-use recency yet. - self.epochs[slot] = 0; - } - } - - /// The device lock separately proves the physical graph was never published. - /// An open owner keeps another usable pair. Exclude every live borrower, - /// including off-lock sibling construction; a closed owner needs no spare. - fn reassign_unused_spare( - &mut self, - slot: u8, - expected: Option, - owner: Option, - ) -> bool { - let slot = usize::from(slot); - if slot == 0 - || slot >= cfg::RENDER_SLOTS - || owner.is_some_and(|owner| { - owner == 0 - || self.owners.iter().filter(|value| **value == owner).count() - >= cfg::RENDER_PAIRS_PER_OWNER - }) - { - return false; - } - let bit = 1u64 << slot; - let old = self.owners[slot]; - if old == 0 - || self.killed & bit != 0 - || owner == Some(old) - || expected.is_some_and(|expected| expected != old) - || self.refs[slot] != 0 - || (self.active | self.quarantined | self.kill_hold) & bit != 0 - || (self.released & bit == 0 - && !self.owners.iter().enumerate().any(|(other, value)| { - other != slot - && *value == old - && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 - })) - { - return false; + self.advance(slot); } - self.owners[slot] = owner.unwrap_or(0); - self.released &= !bit; - self.epochs[slot] = 0; - true } fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { if self.refs.get(usize::from(publishing)) != Some(&1) { @@ -287,31 +188,9 @@ impl State { && self.owners[*slot] == owner && self.refs[*slot] == 0 && self.quarantined & (1u64 << slot) == 0 - && self.released & (1u64 << slot) == 0 }) .map(|slot| slot as u8) } - fn borrow_warm_sibling(&mut self, owner: u64, publishing: u8) -> Option { - let slot = self.warmable_sibling(owner, publishing)?; - self.refs[usize::from(slot)] += 1; - self.active |= 1u64 << slot; - Some(slot) - } - fn cancel_empty_slot(&mut self, slot: u8, owner: u64) -> bool { - let slot = usize::from(slot); - let bit = 1u64 << slot; - if self.owners[slot] == owner - && self.refs[slot] == 1 - && (self.quarantined | self.kill_hold) & bit == 0 - { - self.owners[slot] = 0; - self.released &= !bit; - self.epochs[slot] = 0; - true - } else { - false - } - } fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { let Some(request) = request else { return Plan::Any; @@ -334,28 +213,6 @@ impl State { ) } - /// Whether `pick` would return a slot, without taking it. - fn can_pick(&self, owner: u64, plan: Plan) -> bool { - let usable = |slot: usize| { - self.owners[slot] == owner - && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 - && self.refs[slot] < cfg::RENDER_DEPTH - }; - match plan { - Plan::Wait => false, - Plan::Pair(slot) => usize::from(slot) < cfg::RENDER_SLOTS && usable(usize::from(slot)), - Plan::Any if owner == 0 => false, - Plan::Any if self.owners.contains(&owner) => (0..cfg::RENDER_SLOTS).any(usable), - Plan::Any => { - let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; - (0..cfg::RENDER_SLOTS).any(|slot| { - free & (1u64 << slot) != 0 - && (self.released & (1u64 << slot) != 0 || self.owners[slot] == 0) - }) - } - } - } - fn pick(&mut self, owner: u64, plan: Plan) -> Option { match plan { Plan::Any => self.acquire(owner), @@ -394,7 +251,6 @@ impl State { slot, ticket, footprint: request.footprint.clone(), - fragment: None, }, GFP_KERNEL, ) @@ -402,12 +258,6 @@ impl State { Some((owner, request.sequence)) } - fn position(&self, slot: u8, (owner, sequence): (u64, u64)) -> Option { - self.footprints.iter().position(|entry| { - entry.slot == Some(slot) && entry.owner == owner && entry.sequence == sequence - }) - } - fn forget_waiter(&mut self, ticket: Option) { if let Some(ticket) = ticket { self.footprints @@ -433,55 +283,23 @@ pub(crate) struct Pool { #[pin] released: CondVar, queues: AtomicU32, - /// Logical queues admitted at most: each must be able to hold a render pair. - max_queues: u32, - /// Logical-queue creation waiting for a returned slot. - waiters: Option>, - /// Render publication waiting for render capacity: woken with every - /// release event above. - renders: Option>, } impl Pool { - #[cfg(CONFIG_KUNIT = "y")] - pub(crate) fn new(max_queues: u32) -> Result> { - Self::with_waiters(max_queues, None, None) - } - pub(crate) fn with_waiters( - max_queues: u32, - waiters: Option>, - renders: Option>, - ) -> Result> { - if max_queues > MAX_EXECUTION_CONTEXTS { - return Err(EINVAL); - } + pub(crate) fn new() -> Result> { Arc::pin_init( pin_init!(Self { state <- new_mutex!(State::new(), "G17 render admission"), released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), - max_queues, - waiters, - renders, }), GFP_KERNEL, ) } - /// Every pair or lease return, owner close and quarantine change. - fn returned(&self) { - self.released.notify_all(); - if let Some(renders) = &self.renders { - renders.notify(); - } - } - /// Logical queues currently counted, including one being created. - pub(crate) fn queues(&self) -> u32 { - self.queues.load(Ordering::Acquire) - } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { - if count >= self.max_queues { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::LogicalQueues)); + if count >= MAX_EXECUTION_CONTEXTS { + return Err(ENOSPC); } match self.queues.compare_exchange( count, @@ -494,36 +312,20 @@ impl Pool { } } } - /// The owner's pair is quarantined behind another VM's recovery (spared) and drains. - pub(crate) fn spared_quarantined(&self, owner: u64) -> bool { - let state = self.state.lock(); - state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & state.spared & (1u64 << slot) != 0) - } - pub(crate) fn acquire_render( self: &Arc, owner: u64, request: Option>, - deadline_ns: u64, - check: impl Fn() -> Result, ) -> Result { if owner == 0 { return Err(EINVAL); } let mut state = self.state.lock(); - let started = crate::g17::now_ns(); - let conflict_deadline = started.saturating_add(u64::from(hazard::WAIT_MS) * 1_000_000); - let quarantine_deadline = started.saturating_add(u64::from(QUARANTINE_WAIT_MS) * 1_000_000); + let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); + let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); let mut expired = false; let mut ticket = None; let result = (|| loop { - check()?; - let now = crate::g17::now_ns(); - let remaining_ms = wait_slice_ms(deadline_ns, now)?; if state.exhausted() { return Err(EIO); } @@ -535,31 +337,20 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - // Behind an innocent pair's quarantine (another VM wedged the GPU) wait for the - // pair to drain until the caller's deadline, which the caller extends while it - // drains (RECOVERY_WAIT_MS); a guilty pair's publication fails after a second. - let spared = state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.spared & (1u64 << slot) != 0); - if now >= quarantine_deadline && !spared { + if quarantine_budget == 0 { return Err(EIO); } - let slice = if spared { - remaining_ms - } else { - remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?) + quarantine_budget = match self + .released + .wait_interruptible_timeout(&mut state, quarantine_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, }; - if matches!(self.released.wait_interruptible_timeout( - &mut state, kernel::time::msecs_to_jiffies(slice) - ), CondVarTimeoutResult::Signal { .. }) { - return Err(ERESTARTSYS); - } continue; } state.refusal(owner)?; - expired |= now >= conflict_deadline; let plan = if expired { Plan::Any } else { @@ -585,17 +376,18 @@ impl Pool { } } } - } - // Keep the established hazard wait/fallback rule, but cap every - // sleep with the caller's original deadline and cancellation poll. - let slice = if plan != Plan::Any { - remaining_ms.min(wait_slice_ms(conflict_deadline, now)?) - } else { - remaining_ms - }; - if matches!(self.released.wait_interruptible_timeout( - &mut state, kernel::time::msecs_to_jiffies(slice) - ), CondVarTimeoutResult::Signal { .. }) { + match self + .released + .wait_interruptible_timeout(&mut state, conflict_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => { + conflict_budget = 0; + expired = true; + } + CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, + } + } else if self.released.wait_interruptible(&mut state) { return Err(ERESTARTSYS); } })(); @@ -628,51 +420,8 @@ impl Pool { record, })) } - /// Whether `acquire_render` would admit this request now, without - /// admitting it: the scheduler's prepare step waits for this. - pub(crate) fn admits_render(&self, owner: u64, request: Option>) -> bool { - let state = self.state.lock(); - if owner == 0 || state.refusal(owner).is_err() { - return false; - } - if state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) - { - return false; - } - let plan = state.plan(owner, request, None); - state.can_pick(owner, plan) - } - pub(crate) fn warmable_sibling( - self: &Arc, - owner: u64, - publishing: u8, - ) -> Option { - let mut state = self.state.lock(); - let slot = state.borrow_warm_sibling(owner, publishing)?; - // This is a host construction borrower, not a GPU-use epoch. In - // particular it must not clear a logical close or age an unused spare. - Some(RenderLease { - pool: self.clone(), - slot, - record: None, - }) - } - pub(crate) fn reassign_unused_spare( - &self, - slot: u8, - expected: Option, - owner: Option, - ) -> bool { - self.state - .lock() - .reassign_unused_spare(slot, expected, owner) - } - pub(crate) fn owns_slot(&self, owner: u64, slot: usize) -> bool { - self.state.lock().owners.get(slot) == Some(&owner) + pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { + self.state.lock().warmable_sibling(owner, publishing) } pub(crate) fn release_owner(&self, owner: u64) { let mut state = self.state.lock(); @@ -682,7 +431,7 @@ impl Pool { } } drop(state); - self.returned(); + self.released.notify_all(); } /// Active leases survive this transition. Their later drop is still the /// event which releases capacity, so a stopped owner is never double-freed. @@ -697,97 +446,9 @@ impl Pool { } state.released &= !released; state.quarantined &= !released; - state.spared &= !released; state.kill_hold = 0; - state.killed = 0; drop(state); - self.returned(); - } - - /// Whether the physical queue table can hold another pair (see `State::build_room`). - pub(crate) fn note_build_room(&self, room: bool) { - let mut state = self.state.lock(); - if state.build_room != room { - state.build_room = room; - drop(state); - self.returned(); - } - } - - /// A killed context's pair is free for a new owner. - pub(crate) fn killed_pair_free(&self) -> bool { - let state = self.state.lock(); - state.killed & state.released & !state.active & !state.quarantined & !state.kill_hold & State::MASK != 0 - } - -} - -#[kunit_tests(asahi_g17_admission_accounting)] -mod accounting_tests { - use super::*; - - #[test] - fn every_logical_owner_fits_without_speculative_spares() { - let mut state = State::new(); - for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { - let slot = state.acquire(owner).unwrap(); - assert_eq!(state.owners.iter().filter(|id| **id == owner).count(), 1); - state.release(slot); - } - assert_eq!(state.active, 0); - assert_eq!(state.owners.iter().filter(|id| **id != 0).count(), MAX_EXECUTION_CONTEXTS as usize); - } - - #[test] - fn close_does_not_reuse_an_active_lease() { - let mut state = State::new(); - let slot = state.acquire(1).unwrap(); - state.released |= 1u64 << slot; - assert!(!state.reassign_unused_spare(slot, Some(1), None)); - state.release(slot); - assert!(state.reassign_unused_spare(slot, Some(1), None)); - assert_eq!(state.owners[usize::from(slot)], 0); - assert_eq!(state.refs[usize::from(slot)], 0); - assert_eq!(state.active & (1u64 << slot), 0); - } - - #[test] - fn warming_quarantine_and_exit_hold_exclude_reassignment() { - let mut state = State::new(); - let primary = state.acquire(1).unwrap(); - // A retained legacy sibling still obeys all quarantine/kill guards. - state.owners[2] = 1; - let spare = state.borrow_warm_sibling(1, primary).unwrap(); - assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); - state.release(spare); - state.quarantined |= 1u64 << spare; - assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); - state.quarantined = 0; - state.kill_hold |= 1u64 << spare; - assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); - state.kill_hold = 0; - assert!(!state.reassign_unused_spare(spare, Some(3), Some(2))); - assert!(state.reassign_unused_spare(spare, Some(1), Some(2))); - assert!(!state.reassign_unused_spare(primary, Some(1), None)); - state.release(primary); - assert!(!state.reassign_unused_spare(primary, Some(1), None)); - } - - #[test] - fn logical_queue_count_returns_on_drop() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let mut leases = KVec::with_capacity(MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - for _ in 0..MAX_EXECUTION_CONTEXTS { - leases.push(pool.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); - } - assert!(matches!(pool.reserve_queue(), Err(ENOSPC))); - assert_eq!(pool.queues.load(Ordering::Acquire), MAX_EXECUTION_CONTEXTS); - drop(leases.pop()); - let replacement = pool.reserve_queue().unwrap(); - drop(leases); - assert_eq!(pool.queues.load(Ordering::Acquire), 1); - drop(replacement); - assert_eq!(pool.queues.load(Ordering::Acquire), 0); + self.released.notify_all(); } } @@ -795,18 +456,9 @@ mod accounting_tests { pub(crate) struct QueueSlot { pool: Arc, } -impl QueueSlot { - /// The device's admission pool, which lives as long as the firmware session. - pub(crate) fn pool(&self) -> &Arc { - &self.pool - } -} impl Drop for QueueSlot { fn drop(&mut self) { self.pool.queues.fetch_sub(1, Ordering::AcqRel); - if let Some(waiters) = &self.pool.waiters { - waiters.notify(); - } } } @@ -816,123 +468,41 @@ pub(crate) struct RenderLease { record: Option<(u64, u64)>, } impl RenderLease { - /// The device lock proves this slot has no physical identities or graph. - /// Give back an unbound reservation before retrying on a retained channel. - pub(crate) fn cancel_empty(self, owner: u64) -> bool { - let cancelled = { - let mut state = self.pool.state.lock(); - state.cancel_empty_slot(self.slot, owner) - }; - drop(self); - cancelled - } - pub(crate) fn slot(&self) -> u8 { self.slot } - /// `spared`: the failing work is innocent. Any guilty failure makes the slot's quarantine - /// guilty for good; only a quarantine that starts spared and stays spared is marked spared. - pub(crate) fn quarantine(&self, spared: bool) { - let bit = 1u64 << self.slot; - let mut state = self.pool.state.lock(); - if !spared { - state.spared &= !bit; - } else if state.quarantined & bit == 0 { - state.spared |= bit; - } - state.quarantined |= bit; - drop(state); - self.pool.returned(); + pub(crate) fn quarantine(&self) { + self.pool.state.lock().quarantined |= 1u64 << self.slot; + self.pool.released.notify_all(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. pub(crate) fn rehabilitate_after_processor_stop(&self) { - let mut state = self.pool.state.lock(); - state.quarantined &= !(1u64 << self.slot); - state.spared &= !(1u64 << self.slot); + self.pool.state.lock().quarantined &= !(1u64 << self.slot); } /// Requires the runtime's closed-recovery, exact pair-drain witness. pub(crate) fn rehabilitate_after_firmware_drain(&self) { self.rehabilitate_after_processor_stop(); } } -impl RenderLease { - /// Called under the device mutex once the render's kicks are committed, - /// including a replay, which supersedes the earlier kick. - pub(crate) fn note_publication(&self, fragment: KickDependency) { - let Some(record) = self.record else { - return; - }; - let mut state = self.pool.state.lock(); - if let Some(index) = state.position(self.slot, record) { - state.footprints[index].fragment = Some(fragment); - } - } -} - -impl Pool { - /// Under the device mutex, which orders every publication: the newest - /// published fragment kick of each other pair whose live same-VM render - /// overlaps `footprint`. Quarantined pairs are skipped as in admission. - pub(crate) fn publication_barriers( - &self, - owner: u64, - slot: u8, - footprint: &Footprint, - barriers: &mut [KickDependency; hazard::PUBLICATION_BARRIERS], - ) -> Result { - let state = self.state.lock(); - let mut slots = [0u8; hazard::PUBLICATION_BARRIERS]; - let mut count = 0; - for entry in state.footprints.iter() { - let (Some(other), Some(fragment)) = (entry.slot, entry.fragment) else { - continue; - }; - if other == slot - || entry.owner == owner - || state.quarantined & (1u64 << other) != 0 - || !footprint.same_vm(&entry.footprint) - || !footprint.overlaps(&entry.footprint) - { - continue; - } - match slots[..count].iter().position(|known| *known == other) { - // One fragment ring completes in kick order; same QID, so the - // word orders by timestamp. - Some(index) => { - if fragment.word() > barriers[index].word() { - barriers[index] = fragment; - } - } - // Admission bounds the pairs; see hazard::plan. - None if count == hazard::PUBLICATION_BARRIERS => return Err(EBUSY), - None => { - slots[count] = other; - barriers[count] = fragment; - count += 1; - } - } - } - Ok(count) - } -} - impl Drop for RenderLease { fn drop(&mut self) { let mut state = self.pool.state.lock(); - if let Some(record) = self.record { - if let Some(index) = state.position(self.slot, record) { + if let Some((owner, sequence)) = self.record { + if let Some(index) = state.footprints.iter().position(|entry| { + entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence + }) { state.footprints.swap_remove(index); } } state.release(self.slot); drop(state); - self.pool.returned(); + self.pool.released.notify_all(); } } impl State { - fn hold_for_exit_kill(&mut self, owner: u64, slots: u64, executed: u64) -> bool { + fn hold_for_exit_kill(&mut self, owner: u64, slots: u64) -> bool { if owner == 0 || slots & !Self::MASK != 0 { return false; } @@ -941,10 +511,10 @@ impl State { if slots & bit == 0 { continue; } - // A pair whose commands all ran is held with them (see `Pair::executed_for_kill`). if self.owners[slot] != owner - || (executed & bit == 0 - && (self.refs[slot] != 0 || self.active & bit != 0 || self.released & bit == 0)) + || self.refs[slot] != 0 + || self.active & bit != 0 + || self.released & bit == 0 { return false; } @@ -955,280 +525,17 @@ impl State { } impl Pool { /// Hold the departed owner's idle pairs before publishing its exit record. - /// Successful publication retains these bits until the owner's scheduler release is - /// consumed (`release_killed_slots` at detach), or until both processors stop. - pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64, executed: u64) -> bool { - self.state.lock().hold_for_exit_kill(owner, slots, executed) + /// Successful publication retains these bits until both processors stop. + pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64) -> bool { + self.state.lock().hold_for_exit_kill(owner, slots) } - /// Undo a hold whose control record was not published. + /// Undo only a hold whose control record was not published. pub(crate) fn release_exit_kill_hold(&self, slots: u64) { if slots == 0 { return; } self.state.lock().kill_hold &= !slots; - self.returned(); - } - - /// End the kill hold of `slots` once their killed context's scheduler release was - /// consumed (the firmware no longer names these pairs). A new owner takes them last. - pub(crate) fn release_killed_slots(&self, slots: u64) { - let mut state = self.state.lock(); - let held = state.kill_hold & slots & State::MASK; - if held == 0 { - return; - } - let now = crate::g17::now_ns(); - state.kill_hold &= !held; - state.killed |= held; - for slot in 0..cfg::RENDER_SLOTS { - if held & (1u64 << slot) != 0 { - state.killed_at[slot] = now; - } - } - drop(state); - self.returned(); - } -} - -impl Pool { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), self.max_queues)?; - let state = self.state.lock(); - let retained = state.owners.iter().enumerate().fold(0u64, |bits, (slot, owner)| { - bits | if *owner != 0 { 1u64 << slot } else { 0 } - }) & State::MASK; - let used = (state.active | state.quarantined | state.kill_hold | (retained & !state.released)) & State::MASK; - writeln!(f, "render_slots used={} total={} retained={} active={} quarantined={} kill_held={}", - used.count_ones(), State::MASK.count_ones(), retained.count_ones(), - (state.active & State::MASK).count_ones(), (state.quarantined & State::MASK).count_ones(), - (state.kill_hold & State::MASK).count_ones()) - } -} - -#[kunit_tests(asahi_g17_fault_admission)] -mod fault_capacity_tests { - use super::*; - - #[test] - fn a_guilty_failure_makes_a_spared_quarantine_guilty() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let queue = pool.reserve_queue().unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); - let bit = 1u64 << lease.slot(); - assert!(!pool.spared_quarantined(1)); - lease.quarantine(true); - assert_ne!(pool.state.lock().spared & bit, 0); - assert!(pool.spared_quarantined(1)); - lease.quarantine(false); - assert_eq!(pool.state.lock().spared & bit, 0); - assert!(!pool.spared_quarantined(1)); - // Once guilty, a later spared failure does not make it spared again. - lease.quarantine(true); - assert_eq!(pool.state.lock().spared & bit, 0); - assert_ne!(pool.state.lock().quarantined & bit, 0); - pool.release_owner(1); - lease.rehabilitate_after_firmware_drain(); - drop(lease); - drop(queue); - let state = pool.state.lock(); - assert_eq!(state.quarantined | state.spared, 0); - } - - #[test] - fn closed_faulting_clients_return_leases_after_drain() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - for owner in 1..=512 { - let queue = pool.reserve_queue().unwrap(); - let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); - lease.quarantine(false); - pool.release_owner(owner); - let bit = 1u64 << lease.slot(); - assert_ne!(pool.state.lock().active & bit, 0); - assert_ne!(pool.state.lock().quarantined & bit, 0); - // The runtime, not an error fence, supplies this exact drain witness. - lease.rehabilitate_after_firmware_drain(); - drop(lease); - drop(queue); - let state = pool.state.lock(); - assert_eq!(state.active, 0); - assert_eq!(state.quarantined, 0); - assert_eq!(state.owners.iter().enumerate().filter(|(slot, id)| **id != 0 && state.released & (1u64 << slot) == 0).count(), 0); - assert_eq!(pool.queues.load(Ordering::Acquire), 0); - } - } - - #[test] - fn published_exit_kill_holds_survive_close_and_drain() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); - let bit = 1u64 << lease.slot(); - pool.release_owner(1); - drop(lease); - assert!(pool.hold_render_slots_for_exit_kill(1, bit, 0)); - assert!(!pool.reassign_unused_spare(bit.trailing_zeros() as u8, Some(1), Some(2))); - // A failed (unpublished) control may roll back its hold. Published kills - // keep the existing processor-stop rule; this test must not weaken it. - pool.release_retained_owners_after_processor_stop(); - assert_eq!(pool.state.lock().kill_hold, 0); - } - - #[test] - fn consumed_scheduler_release_ends_the_kill_hold() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); - let slot = lease.slot(); - let bit = 1u64 << slot; - pool.release_owner(1); - drop(lease); - assert!(pool.hold_render_slots_for_exit_kill(1, bit, 0)); - assert!(!pool.reassign_unused_spare(slot, Some(1), Some(2))); - pool.release_killed_slots(bit); - assert_eq!(pool.state.lock().kill_hold, 0); - assert_eq!(pool.state.lock().killed, bit); - assert!(pool.killed_pair_free()); - // The killed context's pair is never reassigned as a spare, and a new owner takes - // a new graph before it. - assert!(!pool.reassign_unused_spare(slot, Some(1), Some(2))); - let next = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); - assert_ne!(next.slot(), slot); - drop(next); - pool.release_retained_owners_after_processor_stop(); - assert_eq!(pool.state.lock().killed, 0); - } - - #[test] - fn killed_pairs_are_taken_last_oldest_first() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - { - let mut state = pool.state.lock(); - for slot in 1..cfg::RENDER_SLOTS { - state.owners[slot] = 100 + slot as u64; - state.released |= 1u64 << slot; - state.killed |= 1u64 << slot; - state.killed_at[slot] = 1000 - slot as u64; - } - // One pair released without a kill. - state.killed &= !(1u64 << 7); - } - let first = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); - assert_eq!(first.slot(), 7); - let second = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); - assert_eq!(usize::from(second.slot()), cfg::RENDER_SLOTS - 1); - assert_eq!(pool.state.lock().killed & (1u64 << second.slot()), 0); - } - - #[test] - fn a_full_queue_table_reuses_killed_pairs_before_unowned_slots() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - { - let mut state = pool.state.lock(); - for slot in 1..8 { - state.owners[slot] = 100 + slot as u64; - state.released |= 1u64 << slot; - state.killed |= 1u64 << slot; - state.killed_at[slot] = 1000 - slot as u64; - } - } - // With room in the queue table a new owner builds a pair in an unowned slot. - let built = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); - assert!(usize::from(built.slot()) >= 8); - // Without room it takes the killed pair released longest ago. - pool.note_build_room(false); - let reused = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); - assert_eq!(reused.slot(), 7); - assert_eq!(pool.state.lock().killed & (1u64 << 7), 0); - // It claims no sibling: neither another killed pair nor an unowned slot. - let state = pool.state.lock(); - for slot in 1..7 { - assert_eq!(state.owners[slot], 100 + slot as u64); - } - assert_eq!(state.owners.iter().filter(|owner| **owner == 2).count(), 1); - } -} - -impl Pool { - pub(crate) fn owner_active(&self, owner: u64) -> bool { - let state = self.state.lock(); - state.owners.iter().zip(state.refs.iter()).any(|(id, refs)| *id == owner && *refs != 0) - } -} - -#[kunit_tests(asahi_g17_shared_render_depth)] -mod shared_depth_tests { - use super::*; - #[test] - fn a_full_pair_waits_for_retirement_instead_of_allocating_a_spare() { - let mut state = State::new(); - let first = state.acquire(1).unwrap(); - for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1), Some(first)); } - assert_eq!(state.acquire(1), None); - assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); - state.release(first); - assert_eq!(state.acquire(1), Some(first)); - } -} - -#[kunit_tests(asahi_g17_admission_deadline)] -mod deadline_tests { - use super::*; - - #[test] - fn all_pairs_busy_no_progress_expires_without_release() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - { - let mut state = pool.state.lock(); - // No pair has room, including the requesting owner's retained pair. - for slot in 1..cfg::RENDER_SLOTS { - state.owners[slot] = slot as u64; - state.refs[slot] = cfg::RENDER_DEPTH; - } - state.active = State::MASK; - } - let start = crate::g17::now_ns(); - let result = pool.acquire_render(1, None, start + 20_000_000, || Ok(())); - assert!(matches!(result, Err(error) if error == EBUSY)); - assert!(crate::g17::now_ns() - start < 500_000_000); - assert!(pool.state.lock().footprints.is_empty()); - } - - #[test] - fn busy_pair_observes_cancellation_without_a_release_event() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - { - let mut state = pool.state.lock(); - for slot in 1..cfg::RENDER_SLOTS { - state.owners[slot] = slot as u64; - state.refs[slot] = cfg::RENDER_DEPTH; - } - state.active = State::MASK; - } - let start = crate::g17::now_ns(); - let result = pool.acquire_render(1, None, start + 2_000_000_000, || { - if crate::g17::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } - }); - assert!(matches!(result, Err(error) if error == ECANCELED)); - assert!(crate::g17::now_ns() - start < 500_000_000); - assert!(pool.state.lock().footprints.is_empty()); - } - - #[test] - fn cancellation_precedes_available_capacity() { - let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); - assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), - Err(error) if error == ECANCELED)); - assert_eq!(pool.state.lock().active, 0); - } - - #[test] - fn unrelated_wake_storm_never_renews_absolute_deadline() { - let deadline = 2_000_000_000; - // Each wake is allowed to report the full jiffy budget remaining. Only - // monotonic time, not that report or a wake count, bounds admission. - for now in (0..deadline).step_by(100_000) { - assert!(wait_slice_ms(deadline, now).unwrap() <= 10); - } - assert_eq!(wait_slice_ms(deadline, deadline), Err(EBUSY)); - assert_eq!(wait_slice_ms(deadline, deadline + 1), Err(EBUSY)); + self.released.notify_all(); } } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 55994066825f8..b78598cfbb760 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -9,13 +9,8 @@ pub(crate) mod retirement; use crate::g17::{ channel::Rings, context::Context, - fw::{ - channels::ControlRecord, - compute::{OperandState, COMPUTE_DESCRIPTOR_SIZE}, - queue::*, - variant::PointerLayout, - }, - object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, + fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + object::{Allocator, CpuMap, KernelObject}, }; use crate::{hw::t8140::dynamic, mmu}; use core::sync::atomic::{fence, Ordering}; @@ -30,21 +25,10 @@ const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; -const NATIVE_WAIT_OFFSET: usize = 0x900; -const NATIVE_WAIT_STRIDE: usize = 0x60; -// A submission can name every physical compute channel. Keep the complete -// wait list inside its own cached slot, including the last retained slot. -const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET - + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE) - .next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; const POINTERS: usize = 0x100; -/// Producer/consumer words live on their own uncached page after the graph, -/// which the firmware reads through a cached mapping (queue record, job-list -/// head and item ring), as native does. -const CURSORS: usize = GRAPH_SIZE; const ITEMS: usize = 0x4000; const STAMP: usize = 0x300; @@ -94,32 +78,24 @@ pub(crate) struct Binding { /// Descriptor ring of one compute QID with its kernel-VM alias. Built without /// the device mutex; installed into a vacant graph before an owner binds. pub(crate) struct DescriptorRing { - // Drop the GPU alias before the retained storage returns to its pool. - low: mmu::KernelMapping, - object: PooledObject, + object: KernelObject, + high: mmu::KernelMapping, } impl DescriptorRing { - /// Firmware reads every descriptor through a cached kernel mapping, as it - /// does for render. The pool keeps that mapping until processor stop; the - /// hardware reads the same pages through an uncached canonical low alias. - pub(crate) fn new(alloc: &Allocator<'_>, objects: &Arc) -> Result { - let size = DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; - let object = objects.allocate( - alloc, - Family::Descriptor, - size, - size, - mmu::PROT_GPU_FW_SHARED_RW, + pub(crate) fn new(alloc: &Allocator<'_>) -> Result { + let page = mmu::UAT_PGSZ as u64; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let object = alloc.lower( + DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, + page, + prot, CpuMap::WriteCombined, )?; - let low = object.alias_in( - alloc.uat.kernel_lower_vm(), - dynamic::LOWER, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - )?; - Ok(Self { low, object }) + let mut high = alloc.uat.geometry().kernel_range(); + high.start += dynamic::KERNEL_OFFSET; + let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + Ok(Self { object, high }) } } @@ -131,80 +107,47 @@ struct Graph { usage: KernelObject, operand: KernelObject, support: KernelObject, - graph: PooledObject, + graph: KernelObject, context_low: KernelObject, context_high: KernelObject, - /// Announcements, which stay uncached for the firmware. records: KernelObject, - /// Configuration, predecessor and completion-wait records at the same - /// offsets, which the firmware reads through a cached mapping. - cached_records: PooledObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, - /// Host words of the pointer block in this firmware build. - pointers: PointerLayout, - /// Layout of the operand-state buffer for the GPU's cores. - operand_state: &'static OperandState, } impl Graph { - fn new( - alloc: &Allocator<'_>, - objects: &Arc, - context: &Context, - pointers: PointerLayout, - operand_state: &'static OperandState, - ) -> Result { + fn new(alloc: &Allocator<'_>, context: &Context) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let ring = DescriptorRing::new(alloc, objects)?; + let ring = DescriptorRing::new(alloc)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(operand_state.size, page, prot, cpu)?; + let operand = alloc.lower(0x14000, page, prot, cpu)?; let mut support = alloc.kernel(0xc000, page, prot, cpu)?; let support_va = support.gpu_va(); support.write(0, support_va + page)?; support.write(0x100, support_va + page + 4)?; support.write(0x110, 0x50u32)?; support.write(mmu::UAT_PGSZ + 4, 1u32)?; - let mut graph = objects.allocate( - alloc, - Family::QueueState, - GRAPH_SIZE + mmu::UAT_PGSZ, - GRAPH_SIZE, - prot, - cpu, - )?; + let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + CURSORS as u64, + pointers_va: graph_va + POINTERS as u64, items_va: graph_va + ITEMS as u64, job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), owner_pid: context.owner_pid(), policy: context.policy(), }); - view.pointers = PointerBlock::new(&pointers)?; + view.pointers = PointerBlock::new(); view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; - graph.initialize::(CURSORS, |view| { - *view = PointerBlock::new(&pointers)?; - Ok(()) - })?; let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; - let cached_records = objects.allocate( - alloc, - Family::QueueConfig, - RECORD_SLOTS * CACHED_RECORD_STRIDE, - RECORD_SLOTS * CACHED_RECORD_STRIDE, - prot, - cpu, - )?; Ok(Self { ring: Some(ring), preempt, @@ -215,11 +158,8 @@ impl Graph { context_low, context_high, records, - cached_records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, - pointers, - operand_state, }) } @@ -281,12 +221,9 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self - .graph - .word(CURSORS + self.pointers.producer)? - .load(Ordering::Relaxed), - consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), - count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), + producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), + consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), + count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), }) } fn stamp(&self) -> Result { @@ -297,12 +234,12 @@ impl Graph { fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; - Ok((ring.object.gpu_va() + offset, ring.low.iova() + offset)) + Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { let ring = self.ring.as_ref().ok_or(EAGAIN)?; ring.object - .map_alias(vm, ring.low.iova(), mmu::PROT_GPU_SHARED_RO) + .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) } /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed @@ -321,7 +258,7 @@ impl Graph { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { let cursors = self.cursors()?; if !cursors.consumed(end, cursors.count)? { - return Err(EAGAIN); + return Err(EBUSY); } } Ok(()) @@ -336,14 +273,10 @@ impl Graph { return Err(EIO); } for index in 0..ITEM_RING_ENTRIES as usize { - self.graph - .dword(ITEMS + index * 8)? - .store(0, Ordering::Relaxed); + self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); - self.graph - .word(CURSORS + self.pointers.producer)? - .store(cursors.consumer, Ordering::Relaxed); + self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); Ok(()) @@ -456,21 +389,11 @@ struct Active { failure_deferred: bool, qos_pending: bool, } -struct Previous> { - binding: Option, - context: C, +struct Previous { + binding: Option, + context: Arc, publication: Option<(u64, u64)>, } -impl Previous { - /// The exact drain witness covers the replacement publication even when - /// its public fence failed. An unpublished replacement still needs rollback. - fn retire_witnessed(previous: &mut Option) { - if previous.as_ref().is_some_and(|p| p.publication.is_some()) { - drop(previous.take()); - } - } -} - /// Per-queue addresses of the bound owner, read under the publication preflight. struct BindingAddresses { preempt: u64, @@ -530,21 +453,12 @@ pub(crate) enum PoolPhase { Vacant, } -/// Builder ownership of one physical release transaction. The preparation -/// lease pins this registry until abort/commit; owner close may remove lookups. -#[derive(Clone, Copy)] -pub(crate) struct Growth { - qid: u8, - phase: PoolPhase, -} - pub(crate) struct Queue { owner: Option, /// Scheduler/work pages of an exited last owner whose context was dropped; /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, - release_sent_ns: Option, growing: bool, growth_blocks: usize, /// A killed queue was recycled: its pool/ring must be vacated and re-armed @@ -557,8 +471,6 @@ pub(crate) struct Queue { /// `None` after an exited owner was detached; a new owner binds afresh. binding: Option, previous: Option, - /// This binding has never crossed the successor visibility boundary. - unpublished_binding: bool, kick: kick::Queue, pool: ComputePool, graph: Graph, @@ -568,24 +480,12 @@ pub(crate) struct Queue { replay_error: Option, replay_prepared: bool, exit: Option, - /// Since when the kill of this failed queue's context waits for other contexts' compute, - /// and whether that wait was reported. - kill_wait: Option<(u64, bool)>, installed: bool, /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, - /// Last configuration record the firmware consumed, and its recovery - /// epoch. A command repeats the record only when it differs. - config_cache: Option<(QueueConfig, u64)>, ordinal: u32, submitted: u32, - /// The stamp of the newest command the firmware reported failed ([`Self::note_fault`]), - /// while it is newer than the stored stamp. - fault_stamp: Option, - /// Prepared packets of the bound owner about to publish (see [`Self::claim`]). - claims: u32, - last_published: u64, last_end: u64, released: bool, retirement_ready: bool, @@ -599,8 +499,6 @@ pub(crate) struct Queue { quarantine_error: Option, failure_status_pending: bool, retirement_proved: bool, - source_readers: Arc, - source_hint: crate::g17::sync_input::ReaderHint, } impl Queue { @@ -608,26 +506,20 @@ impl Queue { /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( alloc: &Allocator<'_>, - objects: &Arc, backing: Backing, owner: u64, context: Arc, - pointers: PointerLayout, - operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state)?; + let mut graph = Graph::new(alloc, &context)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; context.mark_published(); - context.note_compute_binding(true); - crate::g17::trace::bind(crate::g17::trace::COMPUTE, kick.id().qid(), context.id(), 0); Ok(Self { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, - release_sent_ns: None, growing: false, growth_blocks: 0, needs_fresh_backing: false, @@ -635,7 +527,6 @@ impl Queue { free_list_generation: 0, binding: Some(binding), previous: None, - unpublished_binding: true, kick, pool, graph, @@ -645,16 +536,11 @@ impl Queue { replay_error: None, replay_prepared: false, exit: None, - kill_wait: None, installed: false, outer_started: 0, published_config: None, - config_cache: None, ordinal: 0, submitted: 0, - fault_stamp: None, - claims: 0, - last_published: 0, last_end: 0, released: false, retirement_ready: false, @@ -668,44 +554,8 @@ impl Queue { quarantine_error: None, failure_status_pending: false, retirement_proved: false, - source_readers: Arc::new(core::sync::atomic::AtomicU32::new(0), GFP_KERNEL)?, - source_hint: crate::g17::sync_input::ReaderHint::new(), - }) - } - /// The firmware failed the command whose completion stamp is `stamp`. It writes a - /// zero-duration completion record for it, which quarantines the queue, but stores no - /// stamp: without this the quarantine would wait for its retirement witness forever and - /// keep the queue's backing. Commands published after the failed one still need their own - /// stamp or record. - pub(crate) fn note_fault(&mut self, stamp: u32) { - if self - .fault_stamp - .is_none_or(|old| retirement::stamp_newer(stamp, old)) - { - self.fault_stamp = Some(stamp); - } - } - - /// The firmware failed a command of this quarantined queue, and the queue still waits for - /// its retirement witness: commands published after the failed one, which the firmware - /// never runs, or the failed command itself, whose ring entry it does not consume (a ray - /// traversal's, on Neo). Only a kill of the queue's context gives them back. - pub(crate) fn fault_pending(&self) -> bool { - self.fault_stamp.is_some() - && self.quarantined - && self.awaiting_witness - && self.exit.is_none() - } - - /// The stamp that proves retirement: the stored one, or a newer reported fault's. - fn witness_stamp(&self) -> Result { - let stored = self.graph.stamp()?; - Ok(match self.fault_stamp { - Some(fault) if retirement::stamp_newer(fault, stored) => fault, - _ => stored, }) } - pub(crate) fn qid(&self) -> u8 { self.kick.id().qid() } @@ -716,40 +566,16 @@ impl Queue { pub(crate) fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } - /// The first command after an owner change away from `context` has been - /// published and not retired: the queue still runs that scheduler's state. - pub(crate) fn switching_from(&self, context: &Arc) -> bool { - self.previous.as_ref().is_some_and(|previous| { - previous.publication.is_some() && Arc::ptr_eq(&previous.context, context) - }) - } - /// The context these queue records were bound to before an owner change - /// whose first command is not published yet. - pub(crate) fn pending_switch_from(&self) -> Option<&Arc> { - self.previous - .as_ref() - .filter(|previous| previous.publication.is_none()) - .map(|previous| &previous.context) - } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !self - .context() - .is_some_and(|bound| Arc::ptr_eq(bound, context)) + !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) || (self.active.len == 0 && self.replays.len == 0) } pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined } - /// The oldest command is a published long job (ray traversal or AS build) whose - /// completion record the driver has not seen yet. - pub(crate) fn long_job_running(&self) -> bool { - !self.quarantined - && self.active.front().is_some_and(|active| { - active.observed.is_none() - && active.packet.is_published() - && active.packet.command.long_job() - }) + pub(crate) fn room(&self) -> bool { + !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT } /// Quarantines without a pending witness no longer describe live pipe order. pub(crate) fn timeout_head(&self) -> Option { @@ -757,10 +583,9 @@ impl Queue { return None; } self.active.front().map(|active| crate::g17::timeout::Head { - vm: active.packet.context.vm().id(), + vm: Arc::as_ptr(active.packet.completion.status()) as usize, published: active.published, finished: active.observed.is_some(), - failed: self.quarantined, }) } @@ -778,27 +603,6 @@ impl Queue { visit(active.packet.completion.status()); } } - /// A recovery record names this queue and the completion stamp of the - /// command it interrupted, `kick << 8` like the queue's stamp. Without a - /// stamp match the oldest published command is marked. - pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { - let sequence = stamp >> 8; - if let Some(active) = self.active.iter().find(|active| { - active - .ticket - .is_some_and(|ticket| (ticket.kick as u32) & 0xff_ffff == sequence) - }) { - active.packet.completion.note_interrupted(); - return recovery::Interrupted::Exact; - } - match self.active.iter().find(|active| active.ticket.is_some()) { - Some(active) => { - active.packet.completion.note_interrupted(); - recovery::Interrupted::Oldest - } - None => recovery::Interrupted::None, - } - } pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { for active in self.active.iter() { active @@ -812,10 +616,7 @@ impl Queue { qid: self.qid(), // Accounting completes only for bound owners; a detached queue has no // pending work and never reaches this. - qos: self - .binding - .as_ref() - .map_or(0, |binding| binding.context.qos_id()), + qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), data_master: DataMaster::Compute as u8, } } @@ -825,62 +626,38 @@ impl Queue { !self.quarantined && self.retire_pending } - /// A release has no completion event: keep observing the control consumer - /// until its witness lets the idle physical queue be re-armed. - pub(crate) fn pool_release_pending(&self) -> bool { - matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) - } - - pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { - self.pool_release_pending() - && crate::g17::timeout::background_stalled(self.release_sent_ns, now) - } - - /// A closed owner can still retain its last commands or their completion - /// stamps. Admission may wait for retirement, without treating that close - /// as permission to reuse any firmware-owned backing. - pub(crate) fn release_settling(&self) -> bool { - // release_owner keeps the old owner while its last commands retire. - // Waiting only once owner is None misses precisely that interval. - release_pending( - self.released, - self.quarantined, - self.active.len, - self.retire_pending, - ) - } - fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) } - /// Stage every unlatched exact identity before the device's single shared-ring scan, - /// whichever queues the scan's batch retires. - pub(crate) fn request_observations(&self, batch: &mut Batch) -> Result { - if self.quarantined { + /// Stage every unlatched exact identity before the device's single shared-ring scan. + pub(crate) fn request_observations( + &self, + batch: &mut Batch, + masks: Option<[u64; 2]>, + ) -> Result { + if self.quarantined || !self.selected(masks) { return Ok(false); } - batch.stage_frontier(self.qid(), self.kick.timestamp().get())?; let mut requested = false; - let mut stamp = None; for active in self.active.iter() { let Some(ticket) = active.ticket else { break; }; if active.observed.is_none() { - let stamp = match stamp { - Some(stamp) => stamp, - None => *stamp.insert(self.graph.stamp()?), - }; - batch.request(&ticket, retirement::stamp_covers(stamp, ticket.kick))?; + batch.request(&ticket)?; requested = true; } } Ok(requested) } - pub(crate) fn latch_observations(&mut self, batch: &mut Batch) -> Result { - if self.quarantined { + pub(crate) fn latch_observations( + &mut self, + batch: &mut Batch, + masks: Option<[u64; 2]>, + ) -> Result { + if self.quarantined || !self.selected(masks) { return Ok(()); } for active in self.active.iter_mut() { @@ -984,7 +761,6 @@ impl Queue { self.retire_pending = false; let mut oldest = true; while let Some(front) = self.active.front() { - let mut restart = None; let result = (|| { let ticket = front.ticket.ok_or(EIO)?; self.valid_ticket(front, &ticket)?; @@ -1004,18 +780,9 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = !retirement::stamp_covers(stamp, ticket.kick) - && match retirement::stamp_wait(front.observed_at, now_ns(), || { - // A node that cannot be read keeps the bound from the record. - front.work.drained().unwrap_or(true) - }) { - retirement::StampWait::Waiting => false, - retirement::StampWait::Restart(now) => { - restart = Some(now); - false - } - retirement::StampWait::Aged => true, - }; + let record_aged = front + .observed_at + .is_some_and(|at| now_ns().saturating_sub(at) >= retirement::STAMP_GRACE_NS); let timestamps = retirement::poll( &ticket, front.observed, @@ -1047,16 +814,6 @@ impl Queue { self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; - if let Some(ticket) = front.ticket { - crate::g17::recovery::note_retired(); - crate::g17::trace::complete( - crate::g17::trace::COMPUTE, - ticket.qid, - ticket.ordinal, - &front.packet, - [timestamps[0], timestamps[1]], - ); - } self.retire_pending = true; defer(Deferred::Retired( front.packet.completion.clone(), @@ -1072,13 +829,9 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; - self.released_at = now_ns(); } } Ok(None) | Err(EAGAIN) => { - if restart.is_some() { - self.active.iter_mut().next().ok_or(EIO)?.observed_at = restart; - } self.retire_pending = self.front_finished(); break; } @@ -1129,7 +882,7 @@ impl Queue { if !retirement::quarantined_retired( &newest, self.graph.cursors()?, - self.witness_stamp()?, + self.graph.stamp()?, self.active.len as u32, self.submitted, )? { @@ -1173,14 +926,9 @@ impl Queue { drop(self.active.pop()); } self.awaiting_witness = false; - self.fault_stamp = None; self.retirement_proved = false; self.replay_retirement = false; self.spared_deferred = false; - // The stamp and fully consumed ring proved the replacement retired. - // Keeping its predecessor only on the fault path prevents recycling - // forever: neither another retirement nor an exit kill can release it. - Previous::retire_witnessed(&mut self.previous); if self.spared_quarantine { self.spared_quarantine = false; self.quarantine_error = None; @@ -1189,9 +937,15 @@ impl Queue { self.quarantined = false; self.retire_pending = false; self.retirement_ready = true; + if self + .previous + .as_ref() + .is_some_and(|p| p.publication.is_some()) + { + drop(self.previous.take()); + } if self.released { self.owner = None; - self.released_at = now_ns(); } } Ok(()) @@ -1273,245 +1027,6 @@ impl Queue { } } -/// Validate the complete unpublished item budget before changing descriptors, -/// work counts or queue producers. Leave one item empty across ring wrap. -fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { - if cursors.count != ITEM_RING_ENTRIES { - return Err(EIO); - } - if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { - return Err(ERANGE); - } - let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; - if free < if entry_signal { 4 } else { 3 } { - let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); - return Err(EAGAIN); - } - Ok(()) -} - -/// Retained commands are capacity, not a failed VM. Nothing has been written -/// when this check runs, so the outer publisher can wait for retirement under -/// its original admission deadline, just as it does for a full firmware ring. -fn retained_capacity( - quarantined: bool, - released: bool, - active: usize, - replays: usize, - replay_front: bool, -) -> Result { - if quarantined || released { - return Err(EBUSY); - } - if active >= IN_FLIGHT || (replays != 0 && !replay_front) { - return Err(EAGAIN); - } - Ok(()) -} - -/// Logical close promises a future healthy handoff, not immediate reuse. -/// Exact retirement still owns the transition. -fn release_pending(released: bool, quarantined: bool, active: usize, retiring: bool) -> bool { - released && !quarantined && (active != 0 || retiring) -} - -#[kunit_tests(asahi_g17_compute_admission)] -mod admission_tests { - use super::*; - - #[test] - fn complete_native_wait_list_fits_every_cached_slot() { - let waits = crate::g17::kick::COMPUTE_CHANNELS; - for slot in 0..RECORD_SLOTS { - for index in 0..waits { - let start = record_start( - slot, - NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, - NATIVE_WAIT_STRIDE, - CACHED_RECORD_STRIDE, - ) - .unwrap(); - assert!(start >= slot * CACHED_RECORD_STRIDE); - assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); - } - } - } - - #[test] - fn record_window_cannot_overwrite_the_next_slot() { - // The nineteenth wait used to cross the old slot boundary silently. - let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; - assert_eq!( - record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), - Err(EINVAL) - ); - assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); - assert_eq!( - record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), - Err(EINVAL) - ); - assert_eq!( - record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), - Err(EINVAL) - ); - assert_eq!( - record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), - Err(EINVAL) - ); - assert_eq!( - record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), - Ok(CACHED_RECORD_STRIDE - 1) - ); - } - - #[test] - fn retained_ring_pressure_waits_without_discarding_work() { - let mut active = Fifo::new(); - for sequence in 0..IN_FLIGHT { - active.push(sequence).unwrap(); - } - assert_eq!( - retained_capacity(false, false, active.len, 0, false), - Err(EAGAIN) - ); - assert_eq!(active.front(), Some(&0)); - assert_eq!(active.len, IN_FLIGHT); - assert_eq!(active.pop(), Some(0)); - assert_eq!( - retained_capacity(false, false, active.len, 0, false), - Ok(()) - ); - assert_eq!(active.front(), Some(&1)); - } - - #[test] - fn replay_front_and_terminal_ownership_remain_distinct() { - assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); - assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); - assert_eq!( - retained_capacity(false, false, IN_FLIGHT, 1, true), - Err(EAGAIN) - ); - for (quarantined, released) in [(true, false), (false, true), (true, true)] { - assert_eq!( - retained_capacity(quarantined, released, 0, 1, true), - Err(EBUSY) - ); - } - } - - #[test] - fn item_refusal_is_retryable_and_preserves_the_empty_slot() { - let n = ITEM_RING_ENTRIES; - assert_eq!( - item_capacity( - Cursors { - producer: 0, - consumer: 4, - count: n - }, - false - ), - Ok(()) - ); - assert_eq!( - item_capacity( - Cursors { - producer: 0, - consumer: 4, - count: n - }, - true - ), - Err(EAGAIN) - ); - assert_eq!( - item_capacity( - Cursors { - producer: n - 2, - consumer: 2, - count: n - }, - false - ), - Ok(()) - ); - assert_eq!( - item_capacity( - Cursors { - producer: n - 2, - consumer: 2, - count: n - }, - true - ), - Err(EAGAIN) - ); - assert_eq!( - item_capacity( - Cursors { - producer: 10, - consumer: 10, - count: n - }, - true - ), - Ok(()) - ); - } - - #[test] - fn bad_item_cursors_are_not_backpressure() { - let n = ITEM_RING_ENTRIES; - assert_eq!( - item_capacity( - Cursors { - producer: 0, - consumer: 0, - count: 0 - }, - false - ), - Err(EIO) - ); - assert_eq!( - item_capacity( - Cursors { - producer: n, - consumer: 0, - count: n - }, - false - ), - Err(ERANGE) - ); - assert_eq!( - item_capacity( - Cursors { - producer: 0, - consumer: n, - count: n - }, - true - ), - Err(ERANGE) - ); - } - - #[test] - fn closed_owner_waits_for_its_last_command() { - assert!(release_pending(true, false, 1, false)); - assert!(release_pending(true, false, 0, true)); - assert!(!release_pending(true, false, 0, false)); - } - - #[test] - fn live_or_quarantined_owners_do_not_promise_capacity() { - assert!(!release_pending(false, false, 1, true)); - assert!(!release_pending(true, true, 1, true)); - } -} - impl Queue { pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { if self.owner != Some(owner) { @@ -1531,9 +1046,7 @@ impl Queue { Ok(true) } pub(crate) fn reusable(&self) -> bool { - if self.growing { - return false; - } + if self.growing { return false; } self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) @@ -1549,15 +1062,11 @@ impl Queue { && self.active.len == 0 && self.replays.len == 0 && self.previous.is_none() - // Borrow acquisition is serialized by the device mutex. Avoid - // touching the separate loan counter for an otherwise busy queue. - && !self.source_hint.has_readers(&self.source_readers) } pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { if !self.reusable() { return Err(EBUSY); } - self.source_hint.reset_for_owner(&self.source_readers)?; if !context.is_current() { return Err(EFAULT); } @@ -1597,15 +1106,7 @@ impl Queue { self.retained_owner = None; } context.mark_published(); - context.note_compute_binding(true); - crate::g17::trace::bind( - crate::g17::trace::COMPUTE, - self.qid(), - context.id(), - if same_vm { 2 } else { 1 }, - ); self.owner = Some(owner); - self.unpublished_binding = true; self.released = false; self.retirement_ready = false; self.retired_by_teardown = false; @@ -1614,9 +1115,6 @@ impl Queue { /// True allows the registry to cancel the never-published QID and drop this graph. An /// installed graph remains retained even after the logical owner exits without work. pub(crate) fn release_owner(&mut self) -> Result { - if let Some(binding) = &self.binding { - binding.context.note_compute_binding(false); - } if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { return Ok(true); } @@ -1636,13 +1134,6 @@ impl Queue { } self.retirement_ready = true; } - // Rebinding an already detached queue has no Previous. If its - // first command is rejected or cancelled before publication, the - // empty active FIFO is still the original idle witness. No future - // completion can turn retirement_ready back on for this owner. - if self.previous.is_none() { - self.retirement_ready = true; - } if self.retirement_ready { self.owner = None; self.released_at = now_ns(); @@ -1671,10 +1162,7 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - self.pool_phase == PoolPhase::Populated - && self.pool.populated() - && (self.reusable() || self.recycled_pending()) - && !self.source_hint.has_readers(&self.source_readers) + (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() } /// A recycled killed queue still holding the backing its killed work used. pub(crate) fn recycled_pending(&self) -> bool { @@ -1732,7 +1220,6 @@ impl Queue { } fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; - self.config_cache = None; self.exit = None; self.quarantined = false; self.quarantine_error = None; @@ -1742,7 +1229,6 @@ impl Queue { self.spared_deferred = false; self.retire_pending = false; self.awaiting_witness = false; - self.fault_stamp = None; self.retirement_proved = false; self.replay_retirement = false; self.replay_error = None; @@ -1768,11 +1254,11 @@ impl Queue { if !self.vacate_candidate() { return Err(EBUSY); } - let record = - ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let record = ControlRecord::FreeListRelease( + self.pool.release_record(self.free_list_generation), + ); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - self.release_sent_ns = Some(crate::g17::now_ns()); fence(Ordering::SeqCst); Ok(()) } @@ -1784,9 +1270,7 @@ impl Queue { consumer: u32, producer: u32, ) -> Result> { - if self.growing { - return Ok(None); - } + if self.growing { return Ok(None); } let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { return Ok(None); }; @@ -1795,18 +1279,11 @@ impl Queue { } // The retained owner binding's client alias pins the backing: detach it // first (the owner has exited; `reusable()` proved no previous binding). - let alias = self - .binding - .as_mut() - .and_then(|binding| binding.pool_alias.take()); + let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); let pages = self.pool.take_pages(); - let descriptors = self - .binding - .as_mut() - .and_then(|binding| binding.descriptors.take()); + let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); self.pool_phase = PoolPhase::Vacant; - self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1821,13 +1298,12 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if !self.can_rearm() { + if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { return Err(EAGAIN); } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; - self.release_sent_ns = None; self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } @@ -1837,50 +1313,30 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. - pub(crate) fn begin_growth( - &mut self, - init: &crate::g17::initdata::InitData, - blocks: usize, - ) -> Result { + pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { if self.growing { - return Err(EAGAIN); + if blocks < self.growth_blocks { return Err(EAGAIN); } + self.growth_blocks = blocks; + return Ok(false); } - if self.quarantined - || self.released - || self.exit.is_some() - || self.active.len != 0 - || self.replays.len != 0 - || self.previous.is_some() + if self.quarantined || self.released || self.exit.is_some() + || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() || (self.installed && !self.retirement_ready) - || self.pool_phase != PoolPhase::Populated - { - return Err(EAGAIN); - } + || self.pool_phase != PoolPhase::Populated { return Err(EAGAIN); } if self.installed { - let record = - ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - self.release_sent_ns = Some(crate::g17::now_ns()); - } else { - self.pool_phase = PoolPhase::Vacant; - self.release_sent_ns = None; - } + } else { self.pool_phase = PoolPhase::Vacant; } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) } - pub(crate) fn finish_growth( - &mut self, - init: &crate::g17::initdata::InitData, + pub(crate) fn finish_growth(&mut self, init: &crate::g17::initdata::InitData, pages: &mut Option, - alias: &mut Option, - blocks: usize, - ) -> Result { - if !self.growing || blocks < self.growth_blocks { - return Err(EAGAIN); - } + alias: &mut Option, blocks: usize) -> Result { + if !self.growing || blocks < self.growth_blocks { return Err(EAGAIN); } if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { let consumer = init.control_consumer()?.load(Ordering::Acquire); let producer = init.control_producer()?.load(Ordering::Acquire); @@ -1888,45 +1344,16 @@ impl Queue { return Err(EAGAIN); } } - if alias.is_none() { - return Err(EINVAL); - } - let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; - // No ownership moves before every fallible descriptor/header operation. - // The old release is consumed; no queue can use this row until commit. - init.initialize_compute_pool_blocks(self.pool.id(), fresh.page_list_va(), blocks)?; - self.pool.replace_pages(pages)?; - let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); + let old_alias = binding.pool_alias.take(); + let old_pages = self.pool.take_pages(); + self.pool.rearm(pages.take().ok_or(EINVAL)?)?; + init.initialize_compute_pool_blocks(self.pool.id(), self.pool.page_list_va()?, blocks)?; + binding.pool_alias = alias.take(); self.pool_phase = PoolPhase::Populated; - self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { - _alias: old_alias, - _pages: pages.take(), - _descriptors: None, - _ring: None, - }) - } - - pub(crate) fn growth_token(&self) -> Option { - self.growing.then_some(Growth { - qid: self.qid(), - phase: self.pool_phase, - }) - } - - /// Transfer an abandoned builder's release back to the event worker. Do not - /// revoke the published release or free backing here: vacancy still requires - /// its exact control-consumer witness. A different transaction is untouched. - pub(crate) fn abort_growth(&mut self, token: Growth) -> Result> { - if !self.growing || self.qid() != token.qid || self.pool_phase != token.phase { - return Ok(None); - } - self.growing = false; - self.growth_blocks = 0; - self.release_owner().map(Some) + Ok(Vacated { _alias: old_alias, _pages: old_pages, _descriptors: None, _ring: None }) } /// Exact host identity remains valid even when several commands share an ioctl ID. @@ -2009,17 +1436,14 @@ impl Queue { } /// Firmware is halted and publication is closed while its target is copied. pub(crate) fn publish_recovery_target(&self) -> Result { - let Some(target) = self.graph.pointers.recovery_target else { - return Ok(()); - }; let producer = self .graph .graph - .word(CURSORS + self.graph.pointers.producer)? + .word(POINTERS + PointerBlock::PRODUCER)? .load(Ordering::Relaxed); self.graph .graph - .word(CURSORS + target)? + .word(POINTERS + PointerBlock::RECOVERY_TARGET)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -2056,7 +1480,9 @@ impl Queue { let ticket = active.ticket.ok_or(EIO)?; let state = crate::g17::job::fence_status(&active.packet.completion.fence()); if state < 0 { - return Err(crate::g17::dependency::PRODUCER_FAILED); + return Err(crate::g17::dependency::inherited_error(Error::from_errno( + state, + ))); } if state > 0 { return Ok(None); @@ -2069,17 +1495,6 @@ impl Queue { } } -/// Validate the individual slot before deriving an offset into its allocation. -/// Allocation-wide checks cannot detect a write into the following live slot. -fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Result { - if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { - return Err(EINVAL); - } - slot.checked_mul(stride) - .and_then(|start| start.checked_add(offset)) - .ok_or(EINVAL) -} - impl Graph { fn record( &mut self, @@ -2089,15 +1504,12 @@ impl Graph { value: T, ) -> Result { let (object, start) = if let Some(slot) = slot { - let (object, stride): (&mut KernelObject, usize) = if offset == ANNOUNCE { - (&mut self.records, RECORD_STRIDE) - } else { - (&mut self.cached_records, CACHED_RECORD_STRIDE) - }; - (object, record_start(slot, offset, window, stride)?) + if slot >= RECORD_SLOTS { + return Err(EINVAL); + } + (&mut self.records, slot * RECORD_STRIDE + offset) } else { - let start = record_start(0, offset, window, GRAPH_SIZE - 0x6800)?; - (&mut *self.graph, 0x6800 + start) + (&mut self.graph, 0x6800 + offset) }; if core::mem::size_of::() > window { return Err(EINVAL); @@ -2123,7 +1535,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(CURSORS + self.pointers.producer)?; + let producer = self.graph.word(POINTERS + 0x40)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -2149,7 +1561,8 @@ pub(crate) trait Host { ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; + fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) + -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; } @@ -2167,17 +1580,6 @@ impl Queue { dependencies: &[crate::g17::fw::kick::KickDependency], host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - self.publish_with_native_waits(packet, dependencies, &[], host, defer) - } - - pub(in crate::g17) fn publish_with_native_waits( - &mut self, - packet: Arc, - dependencies: &[crate::g17::fw::kick::KickDependency], - native: &[crate::g17::sync_input::NativeWait], - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { use crate::g17::fw::{ channels::WorkSlot, @@ -2212,26 +1614,10 @@ impl Queue { let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; - // The first prototype only waits on an installed consumer channel. - // A retry without a published source falls back to completion on CPU. - if first && !native.is_empty() { - return Err(EAGAIN); - } - if native.len() > crate::g17::kick::COMPUTE_CHANNELS { - return Err(EINVAL); - } let entry_signal = first || dependencies .iter() .any(|dependency| dependency.qid() != qid); - item_capacity(self.graph.cursors()?, entry_signal)?; - if !native.is_empty() { - let cursors = self.graph.cursors()?; - let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; - if free < (if entry_signal { 4 } else { 3 }) + native.len() as u32 { - return Err(EAGAIN); - } - } let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -2254,7 +1640,6 @@ impl Queue { kick: kick_timestamp, preempt_va: addresses.preempt, operand_state_va: addresses.operand, - operand: self.graph.operand_state, usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), @@ -2276,10 +1661,6 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; - let cached = self - .config_cache - .and_then(|(config, at)| (at == epoch).then_some(config)); - let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -2301,7 +1682,6 @@ impl Queue { priority: policy.priority(), }; let published = host.next_compute_publication()?; - self.last_published = published; self.active .push(Active { packet: packet.clone(), @@ -2327,7 +1707,7 @@ impl Queue { .object .initialize::( kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write_zeroed(&args), + |descriptor| descriptor.write(&args), )?; if first { self.graph.context_high.write( @@ -2344,25 +1724,17 @@ impl Queue { let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } - let config_va = if send_config { - Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) - } else { - None - }; + let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; let cursors = self.graph.cursors()?; - let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; - for (index, wait) in native.iter().enumerate() { - let record = wait.bytes( - crate::g17::fw::stamp_value(kick_timestamp.get() as u32), - packet.order.sequence, - ); - let address = self.graph.record( - record_slot, - NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, - NATIVE_WAIT_STRIDE, - record, - )?; - wait_vas.push(address, GFP_KERNEL)?; + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < if entry_signal { 4 } else { 3 } { + return Err(ENOSPC); } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; @@ -2371,7 +1743,6 @@ impl Queue { host.prepare_compute_shared()?; packet.mark_published(); visible = true; - self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -2389,7 +1760,7 @@ impl Queue { })?; producer = self .graph - .append(producer, config_va.ok_or(EINVAL)?, || {}) + .append(producer, config_va, || {}) .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; @@ -2398,27 +1769,16 @@ impl Queue { })?; } else { let target = self.pool.reserve_submission()?; - producer = (|| -> Result { - let mut before_descriptor = cursors.producer; - for &wait in &wait_vas { - before_descriptor = self.graph.append(before_descriptor, wait, || { - packet.mark_published(); - visible = true; - })?; - } - self.graph.append(before_descriptor, descriptor_high, || { - packet.mark_published(); - visible = true; + producer = self + .graph + .append(cursors.producer, descriptor_high, || { + packet.mark_published() }) - })() - .or_else(|error| { - if !visible { + .or_else(|error| { self.pool.cancel_unpublished(target)?; - } - Err(error) - })?; + Err(error) + })?; visible = true; - self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -2429,16 +1789,14 @@ impl Queue { policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; - if let Some(config_va) = config_va { - producer = self - .graph - .append(producer, config_va, || {}) - .or_else(|error| { - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; - } + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; } if first { prepared_kick.prepare_compute_install()?; @@ -2468,12 +1826,10 @@ impl Queue { self.outer_started & ring == 0, )?, )?; + let notify_activation = self.outer_started & ring == 0; self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); - if send_config { - self.config_cache = Some((config, epoch)); - } self.ordinal = ordinal; let (accepted, state) = host.epoch()?; if state != 0 || accepted != epoch { @@ -2498,13 +1854,13 @@ impl Queue { previous.publication = Some((packet.order.sequence, epoch)); } } - // Every publication rings its own pipe and priority class, as render - // publication does: pipe 2 (compute) in bits 1:0, priority class in - // bits 3:2. The generic work doorbell left the firmware to find the - // ring itself and delayed dispatch. + if notify_activation { + fence(Ordering::SeqCst); + // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + } fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; - crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); + host.notify((0x83 << 48) | 0x10)?; host.note_submission() })(); if let Err(error) = result { @@ -2542,13 +1898,11 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - retained_capacity( - self.quarantined, - self.released, - self.active.len, - self.replays.len, - replay_front, - )?; + if !self.room() + && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) + { + return Err(EBUSY); + } if !packet.context.is_current() || !self .binding @@ -2615,7 +1969,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(CURSORS + self.pointers.producer)?; + let producer = self.graph.word(POINTERS + 0x40)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); @@ -2628,307 +1982,3 @@ impl Queue { self.terminal.take() } } - -impl Queue { - pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - write!(f, "compute_queue qid={} owned={} released={} reusable={} quarantined={} awaiting_witness={} previous={} exit={} growing={} pool={:?} active={} total={} ", - self.qid(), u8::from(self.owner.is_some()), u8::from(self.released), u8::from(self.reusable()), - u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), - u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; - match self.graph.cursors() { - Ok(cursors) if cursors.count != 0 => writeln!( - f, - "items_used={} items_total={}", - cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, - cursors.count - 1 - ), - _ => writeln!(f, "items_unavailable=1"), - }?; - writeln!( - f, - "compute_settling qid={} release={} pool_release={}", - self.qid(), - u8::from(self.release_settling()), - u8::from(self.pool_release_pending()) - )?; - if let Some(binding) = self.binding.as_ref() { - binding - .context - .write_capacity(f, "compute", usize::from(self.qid()))?; - } - if let Some(previous) = self.previous.as_ref() { - previous - .context - .write_capacity(f, "compute_previous", usize::from(self.qid()))?; - } - Ok(()) - } -} - -#[kunit_tests(asahi_g17_fault_ownership)] -mod fault_ownership_tests { - use super::*; - use core::sync::atomic::AtomicUsize; - - struct Lease(Arc); - impl Lease { - fn new(used: &Arc) -> Self { - used.fetch_add(1, Ordering::Relaxed); - Self(used.clone()) - } - } - impl Drop for Lease { - fn drop(&mut self) { - self.0.fetch_sub(1, Ordering::Relaxed); - } - } - - // This tests the production predecessor ownership operation without firmware - // storage. Runtime tests must separately establish its drain/kill witness. - fn faulting_clients(publication: Option<(u64, u64)>) { - let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); - for _ in 0..512 { - let mut previous = Some(Previous { - binding: Some(Lease::new(&used)), - context: Lease::new(&used), - publication, - }); - assert_eq!(used.load(Ordering::Relaxed), 2); - Previous::retire_witnessed(&mut previous); - assert_eq!(previous.is_none(), publication.is_some()); - if publication.is_none() { - assert_eq!(used.load(Ordering::Relaxed), 2); - // An unpublished replacement returns its original owner on close. - drop(previous.take()); - } - assert_eq!(used.load(Ordering::Relaxed), 0); - Previous::retire_witnessed(&mut previous); - assert_eq!(used.load(Ordering::Relaxed), 0); - } - } - - #[test] - fn faulted_published_replacements_return_to_baseline() { - faulting_clients(Some((1, 7))); - } - - #[test] - fn unpublished_replacements_survive_until_rollback() { - faulting_clients(None); - } - - #[test] - fn same_vm_replacement_without_binding_returns_its_context() { - let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); - let mut previous = Some(Previous:: { - binding: None, - context: Lease::new(&used), - publication: Some((9, 3)), - }); - Previous::retire_witnessed(&mut previous); - assert_eq!(used.load(Ordering::Relaxed), 0); - } -} - -impl Queue { - /// Offer an open client's idle physical channel to another ready job. - /// Logical queue/context ownership is unchanged. The existing binding is - /// retained through the successor's exact replacement witness. - pub(crate) fn share_idle(&mut self) -> Result { - if !self.shareable() { - return Ok(false); - } - self.release_owner()?; - Ok(self.reusable()) - } - - /// Another owner could take this channel now: idle and released, shareable, or vacant. - pub(crate) fn channel_free(&self) -> bool { - self.reusable() || self.shareable() || self.can_rearm() - } - - /// A pool replacement is in flight for this queue. - pub(crate) fn growing(&self) -> bool { - self.growing - } - - /// The queue's first command has been published: its identity and configuration exist. - pub(crate) fn installed(&self) -> bool { - self.installed - } - - /// A packet of the bound owner passed prepare_job and will publish here. - pub(crate) fn claim(&mut self) { - self.claims = self.claims.saturating_add(1); - } - - pub(crate) fn release_claim(&mut self) { - self.claims = self.claims.saturating_sub(1); - } - - /// An open client's idle channel that [`Self::share_idle`] can hand over. - pub(crate) fn shareable(&self) -> bool { - self.claims == 0 - && self.installed - && self.owner.is_some() - && self.retirement_ready - && !self.growing - && !self.quarantined - && !self.awaiting_witness - && self.exit.is_none() - && self.active.len == 0 - && self.replays.len == 0 - && self.previous.is_none() - && self.pool_phase == PoolPhase::Populated - && !self.source_hint.has_readers(&self.source_readers) - } - - /// Device publication order of this channel's newest command: lower is - /// idle longer. - pub(crate) fn last_published(&self) -> u64 { - self.last_published - } -} - -impl Queue { - pub(crate) fn can_rearm(&self) -> bool { - self.pool_phase == PoolPhase::Vacant - && (self.reusable() || self.recycled_pending()) - && !self.source_hint.has_readers(&self.source_readers) - } -} - -impl Queue { - /// A failed first attempt has no future completion to undo its binding. - /// Return the replaced aliases to the caller for off-lock destruction. The - /// caller validates the current preparation epoch and cancels only a true - /// result's never-published QID; installed rings and identities stay intact. - pub(crate) fn rollback_unpublished( - &mut self, - owner: u64, - context: &Arc, - ) -> Result<(bool, Option)> { - if self.owner != Some(owner) - || !self.unpublished_binding - || self - .binding - .as_ref() - .is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) - || self.growing - || self.quarantined - || self.awaiting_witness - || self.exit.is_some() - || self.active.len != 0 - || self.replays.len != 0 - || self.retire_pending - || self.failure_status_pending - || self.pool_phase != PoolPhase::Populated - || self - .previous - .as_ref() - .is_some_and(|previous| previous.publication.is_some()) - { - return Ok((false, None)); - } - if !self.installed { - return Ok((true, None)); - } - let mut retired = None; - if let Some(previous) = self.previous.as_ref() { - self.graph.set_owner(&previous.context)?; - } - if let Some(mut previous) = self.previous.take() { - if let Some(binding) = previous.binding.take() { - retired = self.binding.replace(binding); - } else { - self.binding.as_mut().ok_or(EIO)?.context = previous.context; - } - } - self.unpublished_binding = false; - self.retirement_ready = true; - self.released = true; - self.owner = None; - self.released_at = now_ns(); - Ok((false, retired)) - } -} - -impl Queue { - /// Same immutable singleton identity and current physical epoch. - pub(crate) fn has_input_source( - &self, - source: &Arc, - status: &Arc, - epoch: u64, - ) -> bool { - !self.quarantined - && self.replays.len == 0 - && self.active.iter().any(|active| { - active.ticket.is_some_and(|ticket| ticket.epoch == epoch) - && Arc::ptr_eq(active.packet.completion.status(), status) - && active.packet.completion.is_input_source(source) - }) - } - - pub(crate) fn source_dependency( - &mut self, - source: &crate::g17::sync_input::Source, - status: &Arc, - epoch: u64, - ) -> Result> { - for active in self.active.iter() { - if !active.packet.completion.is_input_source(&source.submission) { - continue; - } - if !Arc::ptr_eq(active.packet.completion.status(), status) { - return Err(EINVAL); - } - if self.quarantined || self.replays.len != 0 { - return Err(EAGAIN); - } - let ticket = active.ticket.ok_or(EAGAIN)?; - if ticket.epoch != epoch { - return Err(EAGAIN); - } - self.valid_ticket(active, &ticket)?; - if active.packet.cancel_requested() { - return Err(ECANCELED); - } - source.submission.source_status()?; - let dependency = crate::g17::fw::kick::KickDependency::new( - self.qid(), - crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?, - ) - .ok_or(EIO)?; - let hold = self.source_hint.hold(&self.source_readers)?; - return Ok(Some(( - active.published, - dependency, - self.graph.support.gpu_va() + STAMP as u64, - u32::from(self.qid()), - hold, - ))); - } - Ok(None) - } - - /// Preparing a possible wait does not create a firmware consumer. Mark it - /// only once the consumer's first item is visible, under the device mutex. - pub(crate) fn note_input_consumer(&self, source: &crate::g17::sync_input::Source) { - for active in self.active.iter() { - if active.packet.completion.is_input_source(&source.submission) { - active.packet.completion.note_firmware_consumer(); - break; - } - } - } -} - -/// Publication, dependency, stamp address, queue and the retained source hold. -pub(crate) type SourceDependency = ( - u64, - crate::g17::fw::kick::KickDependency, - u64, - u32, - crate::g17::sync_input::Hold, -); diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 719a74787ca70..187fd27f8e78e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -83,12 +83,11 @@ impl Transaction { } impl Queue { - /// A running command with ray traversal scratch is never killed. On Neo, killing a + /// A command with ray traversal scratch is never killed. On Neo, killing a /// traversal that stopped making progress either crashed the GPU firmware /// when the queue's pool was then released, or stalled every later render /// when it was retained; left alone, the rest of the GPU kept working. Such - /// a queue stays quarantined and retained until its commands retire. A - /// traversal the firmware reported failed no longer runs (see `fault_pending`). + /// a queue stays quarantined and retained until its commands retire. fn active_traversal(&self) -> bool { self.active.iter().any(|active| { matches!( @@ -102,7 +101,7 @@ impl Queue { && self.released && self.active.len != 0 && self.exit.is_none() - && (!self.active_traversal() || self.fault_pending()) + && !self.active_traversal() && (self.quarantined || self.retired_by_teardown) && age >= if self.quarantined { @@ -111,10 +110,6 @@ impl Queue { CANCELLED_DELAY_NS } } - /// A kill of `context` was published through this queue. - pub(crate) fn exit_published(&self, context: &Arc) -> bool { - self.exit.is_some() && self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) - } pub(crate) fn exit_owner(&self, context: &Arc) -> Option { if !self .active @@ -129,23 +124,6 @@ impl Queue { .and_then(|active| active.ticket.map(|ticket| ticket.owner)) }) } - /// The kill of this queue's failed context waits; returns how long it has waited, and - /// whether this is the first time it crossed `slow`. - pub(crate) fn note_kill_wait(&mut self, now: u64, slow: u64) -> (u64, bool) { - let (since, warned) = self.kill_wait.get_or_insert((now, false)); - let waited = now.saturating_sub(*since); - let crossed = waited >= slow && !*warned; - *warned |= crossed; - (waited, crossed) - } - /// How long the kill of this queue's failed context has been waiting, if it is. - pub(crate) fn kill_waiting(&self, now: u64) -> Option { - self.kill_wait.map(|(since, _)| now.saturating_sub(since)) - } - /// The kill was published: how long it waited, if it did. - pub(crate) fn take_kill_wait(&mut self, now: u64) -> u64 { - self.kill_wait.take().map_or(0, |(since, _)| now.saturating_sub(since)) - } pub(crate) fn record_exit(&mut self, cookie: u64, now: u64) { self.exit = Some(Transaction { cookie, @@ -184,7 +162,6 @@ impl Queue { return false; } exit.phase = Phase::Acknowledged; - crate::g17::trace::teardown(4, cookie, u64::from(self.qid()), 0); true } pub(crate) fn settle_exit( @@ -243,9 +220,6 @@ impl Queue { ))?; drop(self.active.pop()); } - // The acknowledged kill and command settlement replace the ordinary - // drain witness, including a failed first publication after rebinding. - super::Previous::retire_witnessed(&mut self.previous); self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; self.quarantined = true; self.awaiting_witness = false; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index d7f821bbce68d..b4d4bcef24421 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,7 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. -use crate::g17::initdata::{CompletionFold, ComputeCompletion}; +use crate::g17::initdata::ComputeCompletion; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -52,12 +52,6 @@ impl Cursors { } } -/// Whether stamp `a` was stored after stamp `b` (both kick stamps, modulo wrap). -pub(crate) const fn stamp_newer(a: u32, b: u32) -> bool { - let delta = a.wrapping_sub(b); - delta != 0 && delta < 0x8000_0000 -} - /// Outstanding kicks span fewer than half the 24-bit stamp sequence. Zero is a valid wrap. pub(crate) const fn stamp_covers(word: u32, kick: u64) -> bool { let delta = word.wrapping_sub((kick as u32).wrapping_shl(8)); @@ -72,69 +66,21 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone, counted from when its ioctl's work-state node drained. -/// -/// The firmware stores the stamp after the cache maintenance that follows the record, so the -/// stamp, not the record, proves that the command's writes are complete and that its mappings -/// may be released. It stores the stamps of an ioctl's commands only once every kick counted on -/// the ioctl's work-state node has completed: while other commands of the same submit ioctl -/// still run, the stamp is not late, however long they run (a command buffer of a dozen render -/// passes, or a render waiting on another queue). [`stamp_wait`] restarts the bound until the -/// node drains. Only a stamp still missing well past that is treated as never stored. The bound -/// stays far below the scheduler timeout, so such a command cannot fail its VM. -pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; - -// A command that waits out the fallback must still retire well inside the scheduler timeout. -const _: () = assert!(STAMP_FALLBACK_NS * 4 <= crate::g17::job::TIMEOUT_MS as u64 * 1_000_000); - -/// Whether a completion record latched at `observed_at` (host time, ns) has waited out -/// [`STAMP_FALLBACK_NS`] by `now`. -pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { - observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) -} - -/// Where a command's wait for its completion stamp stands; see [`stamp_wait`]. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum StampWait { - /// Within [`STAMP_FALLBACK_NS`] of the record, or no record yet. - Waiting, - /// The bound passed while the ioctl's work-state node still runs kicks: the stamp is not - /// late, and the wait counts again from the given host time. - Restart(u64), - /// The node drained and the stamp is still missing after the bound: retire on the record. - Aged, -} - -/// The stamp wait of a command whose exact record was latched at `observed_at` (host ns) and -/// whose stamp is still missing, at `now`. `drained` reports whether every kick counted on the -/// command's work-state node completed; it is asked only once the bound has passed. -pub(crate) fn stamp_wait(observed_at: Option, now: u64, drained: impl FnOnce() -> bool) - -> StampWait -{ - if !record_aged(observed_at, now) { - StampWait::Waiting - } else if drained() { - StampWait::Aged - } else { - StampWait::Restart(now) - } -} +/// before it retires on the record alone. The firmware stores the stamp after the cache +/// maintenance that follows the record; it occasionally never stores it for a command, most +/// visibly while a render of the same VM is running. +pub(crate) const STAMP_GRACE_NS: u64 = 1_000_000; static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -pub(crate) fn stampless_total() -> u64 { - STAMPLESS.load(core::sync::atomic::Ordering::Relaxed) -} - -/// Counts a command that retired on its record because its completion stamp never arrived; -/// logs the first and every 64th. +/// Counts a command that retired on its record without its completion stamp; logs the first and +/// every 64th. pub(crate) fn note_stampless(qid: u8, kick: u64) { let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; if count == 1 || count % 64 == 0 { pr_info!( - "G17: lifetime total {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", + "G17: {} compute commands retired on their completion record without a completion stamp (latest: queue {} kick {:#x})\n", count, - STAMP_FALLBACK_NS / 1_000_000, qid, kick ); @@ -144,14 +90,7 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has -/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`) in the same recovery -/// generation. Recovery may consume items without completing their writes, so an old record -/// and consumed items cannot replace the stamp across a restart. In both cases its items -/// must be consumed. -#[expect( - clippy::too_many_arguments, - reason = "Retirement validates the ticket against independent firmware observations." -)] +/// been present for [`STAMP_GRACE_NS`] (`record_aged`); in both cases its items must be consumed. pub(crate) fn poll( ticket: &Ticket, observed: Option, @@ -165,9 +104,7 @@ pub(crate) fn poll( if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { return Err(ENODATA); } - let same_generation = !matches!(recordless, Recordless::CrossedRecovery); - let completed = stamp_covers(stamp, ticket.kick) - || (same_generation && observed.is_some() && record_aged); + let completed = stamp_covers(stamp, ticket.kick) || (observed.is_some() && record_aged); if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { return Ok(None); } @@ -203,122 +140,35 @@ pub(crate) fn quarantined_retired( && submitted == pending) } -/// Walks a completion ring of `seen.len()` slots from `start` over the records written since -/// the previous walk: `seen` holds each slot's control word as of the last walk that passed it, -/// so the first slot still holding it is where the firmware writes next. `record` folds one -/// slot; the walk also ends once it reports every request found, or at a requested record -/// still being written, which the next walk reads again. Returns where the next walk starts. -pub(crate) fn walk_completions( - start: usize, - seen: &mut [u64], - mut control: impl FnMut(usize) -> Result, - mut record: impl FnMut(usize) -> Result, -) -> Result { - let slots = seen.len(); - if slots == 0 { - return Err(EINVAL); - } - let start = start % slots; - for step in 0..slots { - let slot = (start + step) % slots; - let word = control(slot)?; - if word == seen[slot] { - return Ok(slot); - } - match record(slot)? { - CompletionFold::Incomplete => return Ok(slot), - CompletionFold::Passed => seen[slot] = word, - CompletionFold::Done => { - seen[slot] = word; - return Ok((slot + 1) % slots); - } - } - } - Ok(start) -} - #[derive(Clone, Copy)] struct Request { descriptor: u64, kick: u64, observed: Option, - covered: bool, - next: u16, -} - -const NO_REQUEST: u16 = u16::MAX; -const _: () = assert!(QIDS * IN_FLIGHT < NO_REQUEST as usize); - -fn request_bucket(qid: usize, kick: u64) -> usize { - qid * IN_FLIGHT + kick as usize % IN_FLIGHT } /// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; /// staging and folding never allocate under the device mutex. pub(crate) struct Batch { requests: KVVec, - /// Per-QID kick buckets; collisions retain the complete identity check. - heads: KVVec, slots: [(usize, usize); QIDS], open: Option, - /// Requests with no observation yet; the scan stops when this reaches zero. - unobserved: usize, - /// Unobserved requests whose queue's completion stamp already covered them when staged. - /// The firmware stores the stamp after the record, so each has a record in the ring. - covered: usize, - /// Per QID, the next kick its queue had not published when the batch was staged - /// ([`NO_FRONTIER`] for a QID without a staged queue). A record at or past it belongs - /// to a command published during the scan, which a later batch stages. - frontiers: [u64; QIDS], } -const NO_FRONTIER: u64 = u64::MAX; -const KICK_MASK: u64 = (1 << 40) - 1; - impl Batch { pub(crate) fn new() -> Result { - let mut heads = KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?; - for _ in 0..QIDS * IN_FLIGHT { - heads.push_within_capacity(NO_REQUEST).map_err(|_| ENOSPC)?; - } Ok(Self { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, - heads, slots: [(0, 0); QIDS], open: None, - unobserved: 0, - covered: 0, - frontiers: [NO_FRONTIER; QIDS], }) } pub(crate) fn clear(&mut self) { self.requests.clear(); self.slots.fill((0, 0)); self.open = None; - self.unobserved = 0; - self.covered = 0; - self.frontiers.fill(NO_FRONTIER); - } - /// Records the first kick `qid`'s queue had not yet published when this batch was staged. - pub(crate) fn stage_frontier(&mut self, qid: u8, next: u64) -> Result { - *self.frontiers.get_mut(usize::from(qid)).ok_or(EINVAL)? = next & KICK_MASK; - Ok(()) - } - /// A record of `qid` with `kick` names a command published after this batch was staged. - fn published_after_staging(&self, qid: usize, kick: u64) -> bool { - let frontier = self.frontiers[qid]; - frontier != NO_FRONTIER && kick.wrapping_sub(frontier) & KICK_MASK < 1 << 39 - } - /// True once every requested identity has an observation. - pub(crate) fn complete(&self) -> bool { - self.unobserved == 0 - } - /// Some request whose record must already be in the ring is still unobserved. - pub(crate) fn covered_pending(&self) -> bool { - self.covered != 0 } - /// Stages `ticket`; `covered` when its queue's completion stamp already covers its kick. - pub(crate) fn request(&mut self, ticket: &Ticket, covered: bool) -> Result { + pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { if ticket.descriptor == 0 || ticket.kick == 0 { return Err(EINVAL); } @@ -330,460 +180,63 @@ impl Batch { return Err(ENOSPC); } let start = if slot.1 == 0 { - let begin = ticket.qid as usize * IN_FLIGHT; - self.heads.get_mut(begin..begin + IN_FLIGHT).ok_or(EIO)?.fill(NO_REQUEST); self.requests.len() } else { slot.0 }; - let bucket = request_bucket(ticket.qid as usize, ticket.kick); - let next = *self.heads.get(bucket).ok_or(EIO)?; - let index = u16::try_from(self.requests.len()).map_err(|_| ENOSPC)?; self.requests .push_within_capacity(Request { descriptor: ticket.descriptor, kick: ticket.kick, observed: None, - covered, - next, }) .map_err(|_| ENOSPC)?; - *self.heads.get_mut(bucket).ok_or(EIO)? = index; *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); - self.unobserved += 1; - self.covered += usize::from(covered); Ok(()) } - /// Capture the control word once. Most slots belong to an older kick of a - /// selected QID; reject those before reading any additional shared words. - pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - if !self.fold_reader(|word| record.read(word))? { - return Ok(CompletionFold::Incomplete); - } - Ok(self.folded()) - } - - /// Like [`Self::fold`], for a scan that reads every slot regardless of the outcome. - pub(crate) fn fold_any(&mut self, record: ComputeCompletion<'_>) -> Result { - self.fold_reader(|word| record.read(word)).map(|_| ()) - } - - fn folded(&self) -> CompletionFold { - if self.complete() { CompletionFold::Done } else { CompletionFold::Passed } - } - - /// Ok(false) when a pending request's record is still being written. - fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { - let control = read(0)?; + /// Capture the control word once, then read only a requested identity's payload. + /// Records can be overwritten during a scan: incomplete or older identities are skipped. + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + let control = record.read(0)?; let qid = ((control >> 9) & 0x7f) as usize; - let (_, count) = self.slots[qid]; - let kick = - (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; + let (start, count) = self.slots[qid]; if count == 0 { - return Ok(!self.published_after_staging(qid, kick)); + return Ok(()); } - let mut index = *self.heads.get(request_bucket(qid, kick)).ok_or(EIO)?; - let mut descriptor = None; - let mut found = None; - for _ in 0..count { - if index == NO_REQUEST { - break; - } - let request = self.requests.get(usize::from(index)).ok_or(EIO)?; - if request.kick == kick { - let value = if let Some(value) = descriptor { - value - } else { - let value = read(6)? & !1; - if value == 0 || read(7)? == 0 { - return Ok(request.observed.is_some()); - } - descriptor = Some(value); - value - }; - if request.descriptor == value { - // Chains run newest first. Retain the original first-match - // rule even if a caller stages the same identity twice. - found = Some(usize::from(index)); - } - } - index = request.next; + let descriptor = record.read(6)? & !1; + if descriptor == 0 || record.read(7)? == 0 { + return Ok(()); } - let Some(index) = found else { - return Ok(!self.published_after_staging(qid, kick)); + let kick = (((control >> 8) & 0xffff_ffff_00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; + let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; + let Some(request) = requests + .iter_mut() + .find(|r| r.descriptor == descriptor && r.kick == kick) + else { + return Ok(()); }; - let request = self.requests.get_mut(index).ok_or(EIO)?; - let timestamps = [read(2)? & TIME_MASK, read(5)? & TIME_MASK]; - if timestamps[0] == 0 || timestamps[1] == 0 { - return Ok(request.observed.is_some()); - } - if request - .observed - .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; + if timestamps[0] == 0 + || timestamps[1] == 0 + || request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) { - return Ok(true); - } - if request.observed.is_none() { - self.unobserved -= 1; - self.covered -= usize::from(request.covered); + return Ok(()); } request.observed = Some(Observation { timestamps }); - Ok(true) + Ok(()) } - - /// A command published after this batch staged its queue's requests was not part of the - /// ring snapshot: it stays unobserved until a later batch. pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; - Ok(self + let request = self .requests .get_mut(start..start + count) .ok_or(EIO)? .iter_mut() .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) - .and_then(|request| request.observed.take())) - } -} - -#[kunit_tests(asahi_g17_compute_stamp_wait)] -mod tests { - use super::*; - - const KICK: u64 = 0x12_3456_7801; - - fn ticket() -> Ticket { - Ticket { - owner: 1, - submission: 1, - context_id: 1, - qid: 28, - descriptor: 0x1000, - work_node: 0x2000, - kick: KICK, - ordinal: 1, - item_producer: 4, - item_count: 64, - epoch: 1, - } - } - - fn consumed() -> Cursors { - Cursors { - producer: 4, - consumer: 4, - count: 64, - } - } - - fn own(kick: u64) -> u32 { - (kick as u32).wrapping_shl(8) - } - - #[test] - fn stamp_covers_its_kick_and_earlier_ones() { - assert!(stamp_covers(own(KICK), KICK)); - assert!(stamp_covers(own(KICK + 1), KICK)); - assert!(!stamp_covers(own(KICK - 1), KICK)); - assert!(!stamp_covers(own(KICK) | 1, KICK)); - assert!(stamp_covers(0, 0xff_ffff)); - } - - #[test] - fn a_reported_fault_stamp_is_newer_only_until_stored() { - assert!(stamp_newer(own(KICK), own(KICK - 1))); - assert!(!stamp_newer(own(KICK), own(KICK))); - assert!(!stamp_newer(own(KICK - 1), own(KICK))); - // Across the wrap of the 24-bit kick in the stamp word. - assert!(stamp_newer(own(0), own(0xff_ffff))); - } - - #[test] - fn a_late_stamp_is_waited_for() { - // Under games on J700 the stamp lands up to about 4 ms after the record while the - // rest of the command's ioctl runs: the command keeps its mappings until then. - let record = Some(Observation { timestamps: [10, 20] }); - let at = 1_000_000; - assert!(!record_aged(Some(at), at + 4_000_000)); - assert!(!record_aged(Some(at), at + 10_000_000)); - let waiting = poll( - &ticket(), - record, - consumed(), - own(KICK - 1), - Recordless::NotYet, - 0, - None, - record_aged(Some(at), at + 10_000_000), - ); - assert_eq!(waiting, Ok(None)); - let stamped = poll( - &ticket(), - record, - consumed(), - own(KICK), - Recordless::NotYet, - 0, - None, - false, - ); - assert_eq!(stamped, Ok(Some([10, 20]))); - } - - #[test] - fn a_later_kick_stamp_retires_it() { - let record = Some(Observation { timestamps: [10, 20] }); - let result = poll( - &ticket(), - record, - consumed(), - own(KICK + 3), - Recordless::NotYet, - 0, - None, - false, - ); - assert_eq!(result, Ok(Some([10, 20]))); - } - - #[test] - fn a_stamp_that_never_comes_retires_on_the_record_after_the_fallback() { - let record = Some(Observation { timestamps: [0x1111, 0x2222] }); - let at = 1_000_000; - assert!(record_aged(Some(at), at + STAMP_FALLBACK_NS)); - assert!(!record_aged(None, u64::MAX)); - let result = poll( - &ticket(), - record, - consumed(), - own(KICK - 1), - Recordless::NotYet, - 0, - None, - record_aged(Some(at), at + STAMP_FALLBACK_NS), - ); - // The record's own timestamps are published, never synthesized ones. - assert_eq!(result, Ok(Some([0x1111, 0x2222]))); - } - - #[test] - fn a_stamp_waits_for_the_rest_of_its_ioctl() { - let at = 1_000_000; - let late = at + STAMP_FALLBACK_NS; - // The node is asked only once the bound passed. - assert_eq!(stamp_wait(Some(at), late - 1, || panic!("node read early")), StampWait::Waiting); - assert_eq!(stamp_wait(None, u64::MAX, || panic!("node read without a record")), - StampWait::Waiting); - // Other kicks of the ioctl still run: the stamp is not late, the bound restarts. - assert_eq!(stamp_wait(Some(at), late, || false), StampWait::Restart(late)); - assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); - // The node drained and the stamp is still missing after the bound. - assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS, || true), StampWait::Aged); - assert_eq!(stamp_wait(Some(at), late, || true), StampWait::Aged); - } - - #[test] - fn an_aged_record_still_waits_for_consumed_items() { - let record = Some(Observation { timestamps: [10, 20] }); - let busy = Cursors { - producer: 4, - consumer: 2, - count: 64, - }; - let result = poll(&ticket(), record, busy, 0, Recordless::Allowed, 0, None, true); - assert_eq!(result, Ok(None)); - } - - #[test] - fn recovery_cannot_turn_an_aged_record_into_a_release_witness() { - let record = Some(Observation { timestamps: [10, 20] }); - // Recovery can advance the consumer without completing cache maintenance. - // Even a record observed before the restart and aged past the fallback - // must retain its command's mappings until a real stamp covers it. - for stamp in [0, own(KICK - 1)] { - assert_eq!(poll(&ticket(), record, consumed(), stamp, - Recordless::CrossedRecovery, 0, None, true), Ok(None)); - } - // A real stamp restores the release witness; completion's interrupted - // classification independently prevents false successful fences. - for stamp in [own(KICK), own(KICK + 1)] { - assert_eq!(poll(&ticket(), record, consumed(), stamp, - Recordless::CrossedRecovery, 0, None, true), Ok(Some([10, 20]))); - } - let busy = Cursors { producer: 4, consumer: 2, count: 64 }; - assert_eq!(poll(&ticket(), record, busy, own(KICK), - Recordless::CrossedRecovery, 0, None, true), Ok(None)); - // The existing same-generation fallback remains available. - assert_eq!(poll(&ticket(), record, consumed(), own(KICK - 1), - Recordless::Allowed, 0, None, true), Ok(Some([10, 20]))); - } - - #[test] - fn a_zero_duration_record_fails() { - let record = Some(Observation { timestamps: [5, 5] }); - let result = poll( - &ticket(), - record, - consumed(), - own(KICK), - Recordless::Allowed, - 0, - None, - true, - ); - assert_eq!(result, Err(ENODATA)); - } -} - - -#[kunit_tests(asahi_g17_completion_index)] -mod index_tests { - use super::*; - - fn ticket(qid: u8, kick: u64, descriptor: u64) -> Ticket { - Ticket { owner: 1, submission: 1, context_id: 1, qid, descriptor, - work_node: 0x2000, kick, ordinal: 1, item_producer: 4, item_count: 64, epoch: 1 } - } - fn record(ticket: &Ticket, timestamps: [u64; 2]) -> [u64; 8] { - let mut words = [0; 8]; - words[0] = ((ticket.kick & !0xff) << 8) | ((ticket.kick & 0xff) << 48) - | (u64::from(ticket.qid) << 9); - words[2] = timestamps[0]; words[5] = timestamps[1]; - words[6] = ticket.descriptor | 1; words[7] = 1; - words - } - #[test] - fn stale_kick_reads_only_the_control_even_on_a_bucket_collision() { - let mut batch = Batch::new().unwrap(); - batch.request(&ticket(7, 33, 0x1000), false).unwrap(); - let words = record(&ticket(7, 17, 0x1000), [10, 20]); - batch.fold_reader(|word| { - assert_eq!(word, 0); - Ok(words[word]) - }).unwrap(); - assert!(!batch.complete()); - } - #[test] - fn collisions_and_reused_descriptors_keep_full_kick_identity() { - let mut batch = Batch::new().unwrap(); - let first = ticket(7, 33, 0x1000); - let second = ticket(7, 49, 0x1000); - batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); - let words = record(&first, [10, 20]); - batch.fold_reader(|word| Ok(words[word])).unwrap(); - assert!(!batch.complete()); - let words = record(&second, [30, 40]); - batch.fold_reader(|word| Ok(words[word])).unwrap(); - assert!(batch.complete()); - assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [10, 20]); - assert_eq!(batch.take(&second).unwrap().unwrap().timestamps, [30, 40]); - } - #[test] - fn same_kick_still_checks_descriptor_and_zero_duration_is_retained() { - let mut batch = Batch::new().unwrap(); - let first = ticket(7, 33, 0x1000); - let second = ticket(7, 33, 0x2000); - batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); - let words = record(&first, [20, 20]); - batch.fold_reader(|word| Ok(words[word])).unwrap(); - assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [20, 20]); - assert!(batch.take(&second).unwrap().is_none()); - } - #[test] - fn duplicate_identity_keeps_the_first_staged_request() { - let mut batch = Batch::new().unwrap(); - let current = ticket(7, 33, 0x1000); - batch.request(¤t, false).unwrap(); batch.request(¤t, false).unwrap(); - let words = record(¤t, [10, 20]); - batch.fold_reader(|word| Ok(words[word])).unwrap(); - assert_eq!(batch.take(¤t).unwrap().unwrap().timestamps, [10, 20]); - assert_eq!(batch.unobserved, 1); - } - #[test] - fn clear_restarts_buckets_and_later_publication_stays_unobserved() { - let mut batch = Batch::new().unwrap(); - batch.request(&ticket(7, 33, 0x1000), false).unwrap(); batch.clear(); - let current = ticket(7, 49, 0x1000); - batch.request(¤t, false).unwrap(); - let stale = record(&ticket(7, 33, 0x1000), [10, 20]); - batch.fold_reader(|word| Ok(stale[word])).unwrap(); - assert!(!batch.complete()); - assert!(batch.take(&ticket(7, 50, 0x2000)).unwrap().is_none()); - let words = record(¤t, [30, 40]); - batch.fold_reader(|word| Ok(words[word])).unwrap(); - assert!(batch.complete()); - } - #[test] - fn a_requested_record_still_being_written_is_read_again() { - let mut batch = Batch::new().unwrap(); - let current = ticket(7, 33, 0x1000); - batch.request(¤t, true).unwrap(); - let mut words = record(¤t, [10, 20]); - words[7] = 0; - assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); - words[7] = 1; - words[5] = 0; - assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); - assert!(batch.covered_pending()); - let words = record(¤t, [10, 20]); - assert!(batch.fold_reader(|word| Ok(words[word])).unwrap()); - assert!(batch.complete() && !batch.covered_pending()); - } - #[test] - fn a_command_published_during_the_scan_is_left_for_the_next_batch() { - let mut batch = Batch::new().unwrap(); - batch.stage_frontier(7, 50).unwrap(); - batch.request(&ticket(7, 49, 0x1000), false).unwrap(); - batch.stage_frontier(9, 3).unwrap(); - let late = record(&ticket(7, 50, 0x3000), [30, 40]); - assert!(!batch.fold_reader(|word| Ok(late[word])).unwrap()); - let idle = record(&ticket(9, 3, 0x3000), [30, 40]); - assert!(!batch.fold_reader(|word| Ok(idle[word])).unwrap()); - let earlier = record(&ticket(9, 2, 0x3000), [30, 40]); - assert!(batch.fold_reader(|word| Ok(earlier[word])).unwrap()); - let wrapped = record(&ticket(7, 0xff_ffff_fff0, 0x3000), [30, 40]); - assert!(batch.fold_reader(|word| Ok(wrapped[word])).unwrap()); - } - fn walk(start: usize, seen: &mut [u64], ring: &[u64], outcome: impl Fn(usize) -> CompletionFold) - -> usize - { - walk_completions(start, seen, |slot| Ok(ring[slot]), |slot| Ok(outcome(slot))).unwrap() - } - #[test] - fn a_walk_reads_only_the_records_written_since_the_previous_one() { - let mut seen = [0u64; 8]; - let mut ring = [0u64; 8]; - assert_eq!(walk(0, &mut seen, &ring, |_| CompletionFold::Passed), 0); - ring[0] = 10; ring[1] = 11; ring[2] = 12; - let read = core::cell::Cell::new(0); - assert_eq!(walk(0, &mut seen, &ring, |_| { read.set(read.get() + 1); CompletionFold::Passed }), 3); - assert_eq!(read.get(), 3); - ring[3] = 13; - assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); - // A full lap later the firmware overwrote slots 4..8 and 0..2 with newer records. - for (slot, word) in [(4, 14), (5, 15), (6, 16), (7, 17), (0, 18), (1, 19)] { ring[slot] = word; } - assert_eq!(walk(4, &mut seen, &ring, |_| CompletionFold::Passed), 2); - } - #[test] - fn a_walk_stops_at_a_partial_record_and_after_the_last_request() { - let mut seen = [0u64; 8]; - let ring = [10, 11, 12, 13, 0, 0, 0, 0]; - assert_eq!(walk(0, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Incomplete } - else { CompletionFold::Passed }), 2); - assert_eq!(seen[..3], [10, 11, 0]); - assert_eq!(walk(2, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Done } - else { CompletionFold::Passed }), 3); - assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); - } - #[test] - fn an_unrequested_or_stale_record_is_passed() { - let mut batch = Batch::new().unwrap(); - batch.request(&ticket(7, 49, 0x1000), false).unwrap(); - let mut stale = record(&ticket(7, 33, 0x1000), [10, 20]); - stale[7] = 0; - assert!(batch.fold_reader(|word| Ok(stale[word])).unwrap()); - let other = record(&ticket(9, 33, 0x1000), [10, 20]); - assert!(batch.fold_reader(|word| Ok(other[word])).unwrap()); - assert!(!batch.complete() && !batch.covered_pending()); + .ok_or(EIO)?; + Ok(request.observed.take()) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index b7037a1c35f75..569b5e289e7d8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -10,8 +10,7 @@ use super::{ command::{Parser, UscWindow, Validated}, completion::{Completion, Destinations, PreparedPublication}, context::Context, - dependency::FailureRecord, - fence::Outputs, + fence::{Outputs, Submission}, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, validation::Proof, }; @@ -28,6 +27,7 @@ use kernel::{ xarray, }; +const EARLY_DEPTH: usize = 2; static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); pub(crate) mod admission; @@ -56,28 +56,21 @@ pub(crate) struct Frontend { allocation_failure: Fence, range: Range, window: UscWindow, - variant: &'static crate::g17::fw::Variant, } impl Frontend { - pub(crate) fn new( - priority: u32, - usc_base: u64, - range: Range, - variant: &'static crate::g17::fw::Variant, - ) -> Result { + pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { if priority > 3 { return Err(EINVAL); } let window = UscWindow::new(usc_base, range.clone())?; - let fences = FenceContexts::new(1, c_str!("aurora_g17_queue"), FENCE_KEY)?; + let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; let allocation_failure = super::fence::allocation_failure(&fences)?; Ok(Self { fences, allocation_failure, range, window, - variant, }) } } @@ -85,7 +78,6 @@ impl Frontend { pub(crate) struct Queue { dev: AsahiDevRef, context: Arc, - usage: Arc, backend: Arc, render: Option>, compute: Option>, @@ -93,24 +85,15 @@ pub(crate) struct Queue { allocation_failure: Fence, range: Range, window: UscWindow, - variant: &'static crate::g17::fw::Variant, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], - /// Finished frontier entries pruned during this ioctl. Their last references - /// can tear a whole completion down; that waits until the ioctl has published. - finished: KVec, - failures: [FailureRecord; 2], + failures: [Option<(u64, Fence)>; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. _slot: admission::QueueSlot, } -type ValidatedBatch = ( - KVec<(Validated, Order, Destinations)>, - KVec, -); - impl Queue { pub(crate) fn new( dev: &AsahiDrmDevice, @@ -118,20 +101,17 @@ impl Queue { backend: Arc, frontend: Frontend, slot: admission::QueueSlot, - usage: Arc, ) -> Result { let Frontend { fences, allocation_failure, range, window, - variant, } = frontend; let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); Ok(Self { dev: dev.into(), context, - usage, backend, render, compute: None, @@ -139,19 +119,16 @@ impl Queue { allocation_failure, range, window, - variant, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), - finished: KVec::new(), - failures: [FailureRecord::new(), FailureRecord::new()], + failures: [None, None], _slot: slot, }) } fn ensure_compute(&mut self, scs_pages: u16) -> Result { - // Validate every request, including an existing scheduler. Physical - // admission happens after its DRM dependencies, in backend publication. + // Admission is checked even when this queue already owns a scheduler. self.backend.ensure_compute(scs_pages)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( @@ -165,40 +142,27 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; - // Without room the entries are released here, as they always could be. - let mut finished = if self - .finished - .reserve(self.frontiers[index].len(), GFP_KERNEL) - .is_ok() - { - Some(&mut self.finished) - } else { - None - }; self.frontiers[index].retain(|(sequence, fences)| { - let status = fences.status(); + let status = job::fence_status(&fences.completed); if status == 0 { return true; } - if let Some(finished) = finished.as_mut() { - let _ = finished.push_within_capacity(fences.clone()); + // One earliest failed ordinal covers every later prefix without + // retaining its packet, mappings, or every failed fence. + if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { + *failure = Some((*sequence, fences.completed.clone())); } - failure.note(*sequence, status, fences); false }); } - #[expect( - clippy::too_many_arguments, - reason = "Enqueue keeps validated command, ordering, and destination owners explicit." - )] fn enqueue( &mut self, id: u64, command: Validated, order: Order, guard: mmu::VmJobGuard, - aggregate: &super::fence::Aggregate, + aggregate: &Arc, inputs: &[file::SyncItem], destinations: Destinations, ) -> Result { @@ -208,62 +172,22 @@ impl Queue { }; let index = engine.index(); if let Validated::Render { ray_query, .. } = &command { - let blocks = self - .backend - .render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; - if blocks > self.context.render_blocks() { - return Err(EINVAL); - } + let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { return Err(EINVAL); } } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { - return Err(EINVAL); - }; + let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; self.ensure_compute(scratch.pages())?; } - // Only convert inputs when no independent host-prefix wait could keep - // the retained source bookkeeping queued behind an external signal. - if engine == Engine::Compute { - super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); - } - let sources = if engine == Engine::Compute - && !inputs.is_empty() - && !destinations.has_outputs() - && self.frontiers[0].iter().all(|(sequence, fences)| { - !Order::contains(order.wait_through[0], *sequence) || fences.status() > 0 - }) - && self.frontiers[1].iter().all(|(sequence, fences)| { - !Order::contains(order.wait_through[1], *sequence) - || fences.ready.raw() != fences.completed.raw() - || fences.status() > 0 - }) - && self.failures.iter_mut().enumerate().all(|(index, failure)| { - failure.earliest().is_none_or(|(sequence, _)| { - !Order::contains(order.wait_through[index], sequence) - }) - }) { - self.backend.input_sources(inputs) - } else { - None - }; - if engine == Engine::Compute { - super::trace::inputs( - self.context.id(), - order.sequence, - 1, - inputs.len(), - sources.as_deref(), - ); - } - // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. let host_predecessors = self.frontiers[index] .iter() .take_while(|(sequence, _)| { - engine == Engine::Render && Order::contains(order.wait_through[index], *sequence) + engine == Engine::Render + && Order::contains(order.wait_through[index], *sequence) }) .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( @@ -275,12 +199,9 @@ impl Queue { destinations, engine == Engine::Compute, host_predecessors.clone().next().is_some(), - self.backend.feed(), - sources.clone(), - self.usage.clone(), + if engine == Engine::Render { self.backend.feed() } else { None }, )?; - let publication = completion - .publication() + let publication = completion.publication() .map(|target| PreparedPublication::new(target, host_predecessors)) .transpose()?; if engine == Engine::Compute { @@ -311,17 +232,14 @@ impl Queue { .iter() .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) .count(); - let mut failures: [Option; 2] = [None, None]; - for (index, failure) in self.failures.iter_mut().enumerate() { - failures[index] = failure.earliest().and_then(|(sequence, fences)| { - Order::contains(order.wait_through[index], sequence).then(|| fences.completed.clone()) - }); - } + let failures: [Option<&Fence>; 2] = core::array::from_fn(|index| { + self.failures[index].as_ref().and_then(|(sequence, fence)| { + Order::contains(order.wait_through[index], *sequence).then_some(fence) + }) + }); let failure_count = failures.iter().flatten().count(); - let mut dependencies = KVec::with_capacity( - inputs.len() + count + timestamp_count + failure_count, - GFP_KERNEL, - )?; + let mut dependencies = + KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; @@ -342,7 +260,7 @@ impl Queue { // Successful completion includes host timestamp publication. Its // ready fence adds no ordering and would prevent shallow inline // publication. Failed and pending producers still need checking. - if fences.status() > 0 { + if job::fence_status(&fences.completed) > 0 { continue; } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; @@ -352,33 +270,13 @@ impl Queue { // either order and must each consume an input's error status before running. for sync in inputs { if let Some(fence) = &sync.fence { - // An input that already signalled without error orders nothing and has no status - // left to consume (like a completed frontier producer above). Keeping it would - // send the render through the scheduler thread instead of inline publication: - // most compositor and client renders carry such already-signalled buffer fences. - if job::fence_status(fence) > 0 { - continue; - } - if sources - .as_ref() - .is_some_and(|sources| sources.contains_input(fence)) - { - continue; - } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } - if let Some(sources) = &sources { - for ready in sources.publication_waits() { - dependencies.push(ready.clone(), GFP_KERNEL)?; - checked_inputs.push(ready.clone(), GFP_KERNEL)?; - } - } - // Publish from the ioctl whenever nothing has to wait, at any depth. - // The backend falls back to the scheduler behind any backlog or on any - // capacity miss, so submission order is preserved. - let early = dependencies.is_empty(); + let early = engine == Engine::Render + && dependencies.is_empty() + && self.frontiers[index].len() < EARLY_DEPTH; let packet = Packet::new( id, self.context.clone(), @@ -392,11 +290,10 @@ impl Queue { Engine::Render => &mut self.render, Engine::Compute => &mut self.compute, }; - let fences = - scheduler - .as_mut() - .ok_or(ENODEV)? - .enqueue(packet, dependencies, early, publication)?; + let fences = scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) @@ -410,7 +307,7 @@ impl Queue { &mut self, bytes: &[u8], objects: Pin<&xarray::XArray>>, - ) -> Result { + ) -> Result<(KVec<(Validated, Order, Destinations)>, KVec)> { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); // All historical ordinals use the same entry counters; publication @@ -423,7 +320,6 @@ impl Queue { self.context.vm(), self.range.clone(), &self.window, - self.variant, &command.payload, )?; let engine = match &payload { @@ -463,10 +359,8 @@ impl Queue { impl crate::queue::Queue for Queue { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - let pages = [ - super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), - super::fw::compute::ScratchRequest::new(layouts[1])?.pages(), - ]; + let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; let blocks = self.backend.render_scs_blocks(pages)?; self.context.reserve_render_blocks(blocks) } @@ -480,28 +374,19 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; + let mut outputs = Outputs::new( + &self.fences, + syncs, + input_count, + &self.allocation_failure, + )?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } - let (mut pending, guards) = self.prepare_submission(bytes, objects)?; - if pending - .iter() - .any(|(command, _, _)| matches!(command, Validated::Render { .. })) - { - self.context.note_render(); - } - if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { - for (command, _, _) in pending.iter_mut() { - command.allow_long_job(); - } - } + let (pending, guards) = self.prepare_submission(bytes, objects)?; let mut guards = guards.into_iter(); - let firmware_source = pending.len() == 1 - && matches!(&pending[0].0, Validated::Compute { .. }) - && !pending[0].2.has_outputs(); - let aggregate = outputs.publish(self.context.clone(), firmware_source)?; + let aggregate = outputs.publish(self.context.clone())?; for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( @@ -516,8 +401,6 @@ impl crate::queue::Queue for Queue { } Ok(()) })(); - // Everything this ioctl publishes is visible; release what it pruned. - self.finished.clear(); if let Err(error) = result { self.context.status().record_if_device_loss(error); if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 2436cfd33ed35..99be455b48e40 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -7,7 +7,7 @@ pub(crate) use observation::Observed; mod dependencies; mod memory; pub(crate) mod parked; -pub(super) mod publication; +mod publication; pub(crate) mod retirement; mod warm; @@ -46,9 +46,6 @@ pub(crate) struct RenderDependencies { pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, - /// Fragment kicks of other pairs' overlapping same-VM renders. - pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], - pub(crate) hazard_count: usize, } impl RenderDependencies { @@ -58,8 +55,6 @@ impl RenderDependencies { tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, - hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], - hazard_count: 0, }; pub(crate) fn render(&self) -> &[KickDependency] { &self.render[..self.render_count] @@ -83,18 +78,15 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - /// Publishes `owners` in order with one pass over the QoS table. - fn qos_publish_pair( + fn qos_publish( &mut self, - owners: [qos::Owner; 2], + owner: qos::Owner, scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result<[qos::Publication; 2]>; + ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - /// The next publication number, shared with compute (timeout ordering). - fn next_publication(&mut self) -> Result; fn publish_outer_pair( &mut self, priority: u8, @@ -128,8 +120,6 @@ struct Prepared { mcache: Option, } struct Active { - /// Publication number, shared with compute (crate::g17::timeout). - published: u64, qos_completed: bool, flist_completed: bool, outer_published: bool, @@ -142,8 +132,6 @@ struct Active { drained_result: Option>, /// First observation of this group retired and unlinked without execution. abandoned_since: Option, - /// First observation of this group executed and consumed while its stamps lagged. - stamp_wait_since: Option, } /// A firmware-retired, unexecuted group must hold on two observations this far @@ -164,11 +152,8 @@ pub(crate) struct Pair { owner: Option, ordinal: u64, initial_packet: u64, - graph_started: bool, clusters: u32, - deflake: abi::Deflake, descriptor_flags: [u32; 2], - variant: &'static crate::g17::fw::Variant, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -183,31 +168,14 @@ pub(crate) struct Pair { previous: Option, displaced: Option, context_update: bool, - /// Retired, never-executed groups awaiting republication on this pair, in - /// their original order. Nothing else is prepared on the pair before them. - replay: Tracker>, - /// When the oldest parked group was parked or the last one republished. - replay_since: Option, - /// Scheduler page of the context of the last published command: the - /// scheduler the firmware last ran these queues for. Zero for a fresh graph. - executed: u64, - /// An owner change in flight: the previous scheduler page and the ordinal of - /// the new owner's first command, until that command leaves the pair. - switch: Option<(u64, u64)>, - /// Completion values of the two kicks of the last publication, kept with the configuration - /// cache: what each queue holds once that publication completed. - published_completions: Option<[u64; 2]>, + /// A retired, never-executed group awaiting republication on this pair. + replay: Option>, } impl Pair { #[inline(never)] - #[expect( - clippy::too_many_arguments, - reason = "The packet constructor spells out the paired render firmware inputs." - )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, - rings: &Arc, ids: [kick::Id; 2], slot: u8, owner: u64, @@ -218,9 +186,7 @@ impl Pair { pm_generation: u64, first: &Arc, clusters: u32, - deflake: abi::Deflake, descriptor_flags: [u32; 2], - variant: &'static crate::g17::fw::Variant, ) -> Result> { if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS @@ -245,10 +211,10 @@ impl Pair { if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters, variant.programs)?; + let layout = target::Layout::new(pass, clusters)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm(), rings)?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; + let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -260,10 +226,9 @@ impl Pair { pm_generation, 0, word, - variant, )?; storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([ta_alias, fragment_alias]), + kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), memory, kicks: [tiling, fragment], free_list, @@ -274,11 +239,8 @@ impl Pair { owner: Some(owner), ordinal: 0, initial_packet: first.id, - graph_started: false, clusters, - deflake, descriptor_flags, - variant, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -289,12 +251,8 @@ impl Pair { terminal: None, previous: None, displaced: None, - replay: Tracker::new(), - replay_since: None, + replay: None, context_update: true, - executed: 0, - switch: None, - published_completions: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -322,29 +280,20 @@ impl Pair { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 || self.replay.len() != 0 + self.active.len() != 0 || self.replay.is_some() } /// A replay or an abandonment grace period needs the worker to poll. pub(crate) fn replay_polling(&self) -> bool { - self.replay.len() != 0 || self.active.iter().any(|a| a.abandoned_since.is_some()) - } - pub(crate) fn replay_stalled(&self, now: u64) -> bool { - self.replay.len() != 0 && crate::g17::timeout::background_stalled(self.replay_since, now) + self.replay.is_some() || self.active.iter().any(|a| a.abandoned_since.is_some()) } pub(crate) fn replay_front(&self) -> Option> { - self.replay.front().cloned() + self.replay.clone() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self - .replay - .front() - .is_some_and(|front| Arc::ptr_eq(front, packet)) - { + if !self.replay.as_ref().is_some_and(|front| Arc::ptr_eq(front, packet)) { return Err(EIO); } - self.replay.pop(); - // Progress: the expiry restarts for the next parked group, if any. - self.replay_since = (self.replay.len() != 0).then(crate::g17::now_ns); + self.replay = None; Ok(()) } pub(crate) fn published_in_flight(&self) -> bool { @@ -366,20 +315,13 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() - || self - .replay - .front() - .is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { return Err(EBUSY); } let Validated::Render { - pass, - attachments, - ray_query, - .. + pass, attachments, ray_query, .. } = &packet.command else { return Err(EINVAL); @@ -389,21 +331,17 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; - self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; - let deflake_layout = self.deflake; + let layout = target::Layout::new(pass, self.clusters)?; let owners = self.memory.owners(); self.memory .target .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; let target = self.memory.target.prepare(ordinal, &layout)?; let scene = self.memory.manager.prepare_scene(ordinal)?; - self.memory - .generation(ordinal)? - .prepare(ordinal, pass, deflake_layout.viewport)?; + self.memory.generation(ordinal)?.prepare(ordinal, pass)?; let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = self.memory.generation(ordinal)?.addresses(); let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { @@ -433,7 +371,6 @@ impl Pair { heapmeta_va: target.heap, tpc_va: self.memory.target.tpc_va(), tpc_size: layout.tpc_size as u64, - gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, fragment_status_va: fragment_status, @@ -465,18 +402,13 @@ impl Pair { stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], }, - programs: self.variant.programs, - deflake: deflake_layout, }; self.memory.graph.claim(ordinal)?; - // Without an MCache table kick-entry bit 58 stays clear. - let attachments = match self.variant.mcache { - true => attachments[1].as_slice(), - false => &[][..], - }; - let result = - self.memory - .write_descriptors(&args, attachments, packet.context.id().try_into()?); + let result = self.memory.write_descriptors( + &args, + attachments[1].as_slice(), + packet.context.id().try_into()?, + ); let mcache = match result { Ok(mcache) => mcache, Err(error) => { @@ -504,7 +436,7 @@ impl Pair { struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, - fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], + fragment: [KickDependency; 3], fragment_count: usize, } @@ -515,7 +447,7 @@ fn render_barriers( ) -> RenderBarriers { let mut tiling_barriers = [KickDependency::ZERO; 3]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; + let mut fragment_barriers = [own_tiling; 3]; let mut fragment_count = 1; for dependency in dependencies.render() { if fragment_stage { @@ -541,11 +473,6 @@ fn render_barriers( tiling_barriers[tiling_count] = compute; tiling_count += 1; } - // Only tile stores overlap: the tiling kick stays concurrent. - for hazard in &dependencies.hazards[..dependencies.hazard_count] { - fragment_barriers[fragment_count] = *hazard; - fragment_count += 1; - } RenderBarriers { tiling: tiling_barriers, tiling_count, @@ -576,17 +503,9 @@ impl Pair { }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; - self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let qids = self.qids(); - let kick_ids = self.kicks.each_ref().map(kick::Queue::id); - // Command preparation assigns context channel state before publishing - // producers. Pin identities before that assignment; warming never - // enters this path and can still give its unused identities back. - for id in kick_ids { - host.publish_qid(id)?; - } let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; @@ -596,19 +515,8 @@ impl Pair { let stamps = host.stamps(qids)?; fence(Ordering::Acquire); let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); - self.memory.completion_links( - ordinal, - kicks, - payload, - self.variant.fragment_busy && !covered[1], - )?; - let leads = match self.variant.lead_commands { - true => { - let init_manager = !self.memory.manager.initialized(); - self.memory.write_leads(ordinal, init_manager)? - } - false => [false; 2], - }; + self.memory + .completion_links(ordinal, kicks, payload, !covered[1])?; let context = &packet.context; let ring = 1u8 << policy.priority(); let qos = context.qos_id(); @@ -666,28 +574,10 @@ impl Pair { } else { [None; 2] }; - // A covered queue holds the completion value of its newest kick. When that kick is the - // previous publication of this pair in this recovery epoch, the seed would reload the - // value already held, so the configuration record is needed only for its identity. - // Each record costs the firmware time before it starts the kick. - let held = self - .published_completions - .take() - .filter(|_| self.recovery_generation == epoch); - let present = [0, 1].map(|stage| { - let mut config = configs[stage]; - let reload = kicks[stage] - .prev() - .completion_value(payload.wrapping_sub(1)); - if config.completion_seed_valid != 0 - && self.installed[stage] - && held.is_some_and(|held| held[stage] == reload) - { - config.completion_seed_valid = 0; - config.completion_seed = 0; - } - publication::required(cache[stage], config) - }); + let present = [ + publication::required(cache[0], configs[0]), + publication::required(cache[1], configs[1]), + ]; let cursors = self.memory.graph.cursors()?; if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); @@ -696,10 +586,7 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let Validated::Render { - pass, ray_query, .. - } = &packet.command - else { + let Validated::Render { pass, ray_query, .. } = &packet.command else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; @@ -732,7 +619,6 @@ impl Pair { abi::TaDescriptor::register_bindings( self.memory.graph.descriptor_client(0, ordinal), ray_query[0].scratch.enabled(), - self.variant.programs, )?, None, ); @@ -742,7 +628,6 @@ impl Pair { abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), ray_query[1].scratch.enabled(), - self.variant.programs, )?, mcache, ); @@ -764,7 +649,6 @@ impl Pair { configs[0], present[0], kicks[0], - leads[0], )?, publication::Lane::new( &memory.graph, @@ -774,7 +658,6 @@ impl Pair { configs[1], present[1], kicks[1], - leads[1], )?, ]; let windows = [lanes[0].window, lanes[1].window]; @@ -802,16 +685,17 @@ impl Pair { windows[0].target().try_into()?, self.outer_started & ring == 0, )?; - let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; + let event_slot = memory + .graph + .support + .word(abi::storage::EVENT_SLOTS + 4)?; let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); let inner = memory.graph.support.word(0x28000)?; let next_inner = (next_ordinal as u32).wrapping_mul(2); let flist_submission = self.free_list.prepare_submission(2)?; let pm_submission = memory.manager.prepare_submission()?; - let published = host.next_publication()?; self.active .push(Active { - published, qos_completed: false, flist_completed: false, outer_published: false, @@ -827,7 +711,6 @@ impl Pair { failed: false, drained_result: None, abandoned_since: None, - stamp_wait_since: None, }) .map_err(|_| EBUSY)?; self.prepared = None; @@ -838,17 +721,18 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - let [fragment, tiling] = host.qos_publish_pair( - [qos_owners[1], qos_owners[0]], - context.scheduler_va(), - policy, - )?; - qos_publications = [Some(tiling), Some(fragment)]; + qos_publications[1] = + Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); + qos_publications[0] = + Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); // A class change happens only at a rebind, after the pair drained. if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; self.registered = Some(policy.priority()); } + for kick in &self.kicks { + host.publish_qid(kick.id())?; + } host.publish_outer_pair( policy.priority(), &fragment_slot, @@ -863,12 +747,7 @@ impl Pair { ) })(); let outer_next = match result { - Ok(next) => { - if leads[0] { - self.memory.manager.mark_initialized(); - } - next - } + Ok(next) => next, Err(error) => { self.quarantined = true; if !packet.completion.spared() && self.terminal.is_none() { @@ -890,9 +769,6 @@ impl Pair { } }; self.ordinal = next_ordinal; - if let Some(fragment) = KickDependency::new(qids[1], kicks[1]) { - packet.completion.note_render_publication(fragment); - } self.cursors = windows.map(|window| window.target()); for stage in 0..2 { if present[stage] { @@ -901,16 +777,10 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; - self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } - let scheduler = packet.context.scheduler_va(); - if self.executed != 0 && self.executed != scheduler { - self.switch = Some((self.executed, ordinal)); - } - self.executed = scheduler; packet.context.mark_published(); self.announce_publication( host, @@ -957,11 +827,7 @@ impl Pair { let policy = packet.context.policy(); // A registration of another class can only follow a rebind, which // requires the pair to have drained. - if self - .registered - .is_some_and(|class| class != policy.priority()) - && self.in_flight() - { + if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { return Err(EIO); } Ok((epoch, prepared.mcache, policy)) @@ -999,12 +865,6 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; - crate::g17::trace::publish( - crate::g17::trace::RENDER, - self.qids()[1], - self.ordinal as u32, - packet, - ); host.note_submission() })(); if let Err(error) = publish { @@ -1057,141 +917,13 @@ impl Pair { }, ] } - /// Bounded first-fault evidence while the recovery owner has firmware - /// halted, before it acknowledges state 1. Never changes retirement. - pub(crate) fn report_recovery(&self, stamps: Result<[u32; 2]>) { - pr_warn!("G17: recovery render slot={} qids={:?} active={} prepared={} quarantined={} stamps={:?} freelist={:?}\n", - self.slot, self.qids(), self.active.len(), self.prepared.is_some(), - self.quarantined, stamps, self.free_list.counters()); - self.memory.manager.report_recovery(); - for active in self.active.iter().take(4) { - if let Validated::Render { pass, .. } = &active.packet.command { - pr_warn!("G17: recovery render geometry ordinal={} width={} height={} layers={} layout={:?}\n", - active.ticket.ordinal, pass.width, pass.height, pass.layers, - target::Layout::new(pass, self.clusters, self.variant.programs)); - } - pr_warn!( - "G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", - active.ticket.ordinal, - active.outer_published, - active.packet.context.id(), - active.packet.order.sequence, - observation::snapshot(&self.memory, active.ticket.ordinal) - ); - } - } - - /// Latches when the front group executed and the firmware consumed its ring entries while - /// `stamps` still lag it, and reports whether that has held for - /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it - /// on its pass record instead of waiting for stamps no later event will move. - pub(crate) fn note_stamp_wait( - &mut self, - stamps: [u32; 2], - state: u32, - now: u64, - ) -> Result { - if self.quarantined || self.prepared.is_some() || state != 0 { - return Ok(false); - } - let Some(front) = self.active.front_mut() else { - return Ok(false); - }; - if !front.outer_published { - return Ok(false); - } - let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; - if stamps - .into_iter() - .all(|stamp| retirement::stamp_covers(stamp, published)) - { - front.stamp_wait_since = None; - return Ok(false); - } - let snapshot = observation::snapshot(&self.memory, front.ticket.ordinal)?; - if !retirement::observe(&front.ticket, snapshot).is_ok_and(|seen| seen.complete) { - front.stamp_wait_since = None; - return Ok(false); - } - let since = *front.stamp_wait_since.get_or_insert(now); - Ok(Self::stamp_wait_expired(front, since, now)) - } - - /// Whether the front's stamp wait latched at `since` has run out by `now`. While other kicks - /// of the group's ioctl still run, its stamps are not late: the wait restarts and the pair - /// stays latched for the rescan. - fn stamp_wait_expired(front: &mut Active, since: u64, now: u64) -> bool { - let drained = || { - // A node that cannot be read keeps the bound from the latch. - front - .packet - .completion - .work_state() - .and_then(|work| work.drained()) - .unwrap_or(true) - }; - match retirement::stamp_wait(since, now, drained) { - retirement::StampWait::Waiting => false, - retirement::StampWait::Restart(now) => { - front.stamp_wait_since = Some(now); - false - } - retirement::StampWait::Expired => true, - } - } - - /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair - /// without a completion notification, which may never come for such a group. - pub(crate) fn stamp_waiting(&self) -> bool { - self.active - .front() - .is_some_and(|front| front.stamp_wait_since.is_some()) - } - - /// The watchdog of `packet` expired while it is this pair's front: if the firmware executed - /// it and consumed its ring entries while the stamps lag, it retires on its pass record now - /// instead of failing. Failing would give up on the stamps as well. - pub(crate) fn expire_stamp_wait( - &mut self, - packet: &Arc, - stamps: [u32; 2], - state: u32, - ) -> Result { - if !self - .active - .front() - .is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) - { - return Ok(false); - } - if !self.note_stamp_wait(stamps, state, crate::g17::now_ns())? && !self.stamp_waiting() { - return Ok(false); - } - if let Some(front) = self.active.front_mut() { - front.stamp_wait_since = Some(0); - } - Ok(true) - } - fn selected(&self, masks: Option<[u64; 2]>) -> bool { - masks.is_none_or(|masks| completion_noted(self.qids(), masks)) - } - /// The oldest published group still running, for cross-VM timeout attribution - /// (crate::g17::timeout). A quarantined pair's work holds the GPU until the firmware drains - /// it, so it stays visible to other VMs' watchdogs, as compute does, marked as failed. - pub(crate) fn timeout_head(&self) -> Option { - self.active - .iter() - .filter(|active| active.outer_published) - .find(|active| active.drained_result.is_none()) - .map(|active| crate::g17::timeout::Head { - vm: active.packet.context.vm().id(), - published: active.published, - finished: false, - failed: self.quarantined, - }) + masks.is_none_or(|masks| { + self.qids() + .into_iter() + .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) + }) } - pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { self.active .iter() @@ -1202,7 +934,7 @@ impl Pair { .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) } pub(crate) fn replay_contains(&self, packet: &Arc) -> bool { - self.replay.iter().any(|replay| Arc::ptr_eq(replay, packet)) + self.replay.as_ref().is_some_and(|replay| Arc::ptr_eq(replay, packet)) } pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { self.active.iter().find_map(|active| { @@ -1223,10 +955,8 @@ impl Pair { let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; - let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; - self.memory - .manager - .note_required(required, crate::g17::now_ns()); + let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); } @@ -1251,28 +981,6 @@ impl Pair { .front() .map(|active| active.packet.completion.spared()) } - /// A recovery record names one of this pair's queues and the completion - /// stamp of the group it interrupted, `(ordinal + 1) << 8` like the pair's - /// stamps. Without a stamp match the oldest published group is marked. - pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { - let sequence = stamp >> 8; - if let Some(active) = self - .active - .iter() - .filter(|active| active.outer_published) - .find(|active| (active.ticket.ordinal.wrapping_add(1) as u32) & 0xff_ffff == sequence) - { - active.packet.completion.note_interrupted(); - return recovery::Interrupted::Exact; - } - match self.active.iter().find(|active| active.outer_published) { - Some(active) => { - active.packet.completion.note_interrupted(); - recovery::Interrupted::Oldest - } - None => recovery::Interrupted::None, - } - } pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { for active in self.active.iter() { visit(active.packet.completion.status()); @@ -1289,10 +997,7 @@ impl Pair { /// A slot stamp must name retained work before its execution markers are meaningful. pub(crate) fn pass_started(&self, stamp: u64) -> Option { const MASK: u64 = (1 << 40) - 1; - let ordinal = self - .active - .iter() - .map(|active| active.ticket.ordinal) + let ordinal = self.active.iter().map(|active| active.ticket.ordinal) .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) } @@ -1304,7 +1009,6 @@ impl Pair { pub(crate) fn resync(&mut self) { self.configs = [None; 2]; - self.published_completions = None; for kick in &mut self.kicks { kick.clear_parent_after_recovery(); } @@ -1324,11 +1028,7 @@ impl Pair { self.quarantined = true; // Successors published after recovery can still fail behind its spared // head. Classify the whole failing group before any fence can settle. - if self - .active - .iter() - .any(|active| active.packet.completion.spared()) - { + if self.active.iter().any(|active| active.packet.completion.spared()) { for active in self.active.iter() { active.packet.completion.classify(recovery::Class::Spared); } @@ -1384,272 +1084,44 @@ impl Pair { let qos = self.qos_owners(); let pool = self.free_list.id(); let buffer = self.buffer_id(); - let Some(front) = self.active.front_mut() else { - return Ok(false); - }; - if !front.outer_published { - return Ok(false); - } - if !Arc::ptr_eq(&front.packet.context, &self.memory.context) - || !self.memory.owns(front.ticket.ordinal) - { - return Err(EIO); - } - let stampless = front - .stamp_wait_since - .is_some_and(|since| Self::stamp_wait_expired(front, since, crate::g17::now_ns())); - let mut observer = Observer { - memory: &self.memory, - qids, - host, - }; - let result = if let Some(observed) = observed { - if observed.ticket != front.ticket { + loop { + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) + || !self.memory.owns(front.ticket.ordinal) + { return Err(EIO); } - if stampless { - retirement::poll_stampless(&mut front.ticket, &mut observer) - } else { + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = if let Some(observed) = observed { + if observed.ticket != front.ticket { + return Err(EIO); + } retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) - } - } else if stampless { - retirement::poll_stampless(&mut front.ticket, &mut observer) - } else { - retirement::poll(&mut front.ticket, &mut observer) - }; - let timestamps = match result { - Ok(retirement::Poll::Complete(timestamps)) => timestamps, - Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer, true), - Ok(retirement::Poll::Retry) => return Ok(false), - Err(error) => { - self.quarantine(error, defer)?; - return Ok(false); - } - }; - let ordinal = front.ticket.ordinal; - let context_update = front.context_update; - if stampless { - retirement::note_stampless(u16::from(self.slot), ordinal); - } - self.note_retirement(ordinal, context_update); - let front = self.active.front_mut().ok_or(EIO)?; - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.qos_completed = true; - front.flist_completed = true; - } - crate::g17::recovery::note_retired(); - crate::g17::trace::complete( - crate::g17::trace::RENDER, - qids[1], - ordinal as u32, - &front.packet, - [timestamps[0], timestamps[3]], - ); - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok(timestamps), - ))?; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - Ok(true) - } - - /// Every retained group was retired and unlinked by firmware without - /// executing (job list empty, stamps cover it, no recovery, fragment end - /// never written): after another VM's recovery the firmware drops an - /// innocent pair's queued groups this way (seen on 25G83 and 26A428). - /// Nothing of them reached their attachments, so the same commands are - /// published again, in order, instead of failing their owners. Only the - /// whole set is replayed; a replayed prefix would run after a live - /// successor. `grace` asks for the strict witness twice ABANDON_GRACE_NS - /// apart (worker polling); the render timeout, long after publication, - /// takes it once as its retirement does. A single sample can still - /// straddle a firmware update of the fragment end record; the cost is - /// that a group which did execute runs a second time (blending twice). - fn replay_abandoned( - &mut self, - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - grace: bool, - ) -> Result { - use retirement::Memory; - if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { - return Ok(false); - } - if let Some(progressed) = self.settle_interrupted_front(host, defer, grace)? { - return Ok(progressed); - } - use crate::g17::replay::{self, Decision, Group}; - let qids = self.qids(); - let memory = &self.memory; - let mut failure = None; - let groups = self.active.iter().map(|active| { - let innocent = - active.outer_published && !active.failed && !active.packet.completion.guilty(); - let blamed = active.packet.completion.blamed(); - let exhausted = - active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; - // decide() stops at the first group that is still live, an error included. - let (abandoned, ran, lost) = if innocent && !blamed { - (|| -> Result<(bool, bool, bool)> { - let mut observer = Observer { - memory, - qids, - host: &mut *host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(active.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - // A torn sample is no proof; a later poll or the timeout looks again. - let witness = |executed| { - matches!( - retirement::abandoned( - &active.ticket, - snapshot, - stamps, - state, - executed - ), - Ok(true) - ) - }; - let unexecuted = witness(false); - // Interrupted work is run again only if it never started, and work that - // another queue waited on never: the wait ended when firmware retired it. - let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) - || active.packet.completion.has_firmware_consumer(); - Ok(( - unexecuted && !lost, - !unexecuted && witness(true), - unexecuted && lost, - )) - })() - .unwrap_or_else(|error: Error| { - failure = Some(error); - (false, false, false) - }) } else { - (false, false, false) + retirement::poll(&mut front.ticket, &mut observer) }; - Group { - innocent, - blamed, - abandoned, - exhausted, - ran, - lost, - } - }); - let decision = replay::decide(groups); - if let Some(error) = failure { - return Err(error); - } - let now = crate::g17::now_ns(); - let front = self.active.front_mut().ok_or(EIO)?; - match decision { - Decision::Refuse => return Ok(false), - Decision::Wait => { - front.abandoned_since = None; - return Ok(false); - } - Decision::Replay | Decision::Fail => {} - } - if grace { - let Some(since) = front.abandoned_since else { - front.abandoned_since = Some(now); - return Ok(false); + let timestamps = match result { + Ok(retirement::Poll::Complete(timestamps)) => timestamps, + Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer), + Ok(retirement::Poll::Retry) => return Ok(false), + Err(error) => { + self.quarantine(error, defer)?; + return Ok(false); + } }; - if now.saturating_sub(since) < ABANDON_GRACE_NS { - return Ok(false); - } - } - if decision == Decision::Fail { - return self.fail_dropped(host, defer); - } - // decide() lets only the front be exhausted. An exhausted front fails, and its - // successors, which consume its output, fail with it rather than run without it. - let mut failing = self.active.front().is_some_and(|front| { - front.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS - }); - if failing { - pr_warn!( - "G17: render slot {}: {} groups retired without execution; replay limit reached, failing them\n", - self.slot, - self.active.len() - ); - } else if !grace || crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { - pr_warn!( - "G17: render slot {}: {} groups retired without execution; replaying\n", - self.slot, - self.active.len() - ); - } - let qos = self.qos_owners(); - let buffer = self.buffer_id(); - let pool = self.free_list.id(); - // Every group parked below shares this expiry (an early return keeps it). - self.replay_since = Some(now); - while let Some(front) = self.active.front_mut() { - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.flist_completed = true; - front.qos_completed = true; - } let ordinal = front.ticket.ordinal; - let packet = front.packet.clone(); - failing = failing || !packet.completion.begin_replay(); - if failing { - if !packet.completion.spared() { - packet.completion.status().record(EIO); - if self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), EIO)); - } - } - defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - continue; - } - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - packet.reset_publication_for_replay(); - self.published_completions = None; - self.replay.push(packet).map_err(|_| EIO)?; - } - Ok(true) - } - - /// Firmware retired every group of the pair after a restart, but some were dropped - /// unexecuted and cannot run again in order (see `replay::decide`). Every group fails now, - /// the ones that ran too: they consumed output the dropped ones never wrote. The work is - /// innocent, so it fails without poisoning its VM, with ENODATA, which its owner sees, - /// instead of waiting for the render timeout to fail it with ETIMEDOUT. - fn fail_dropped( - &mut self, - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - pr_warn!( - "G17: render slot {}: a GPU recovery dropped groups that cannot run again in order; failing {} groups\n", - self.slot, - self.active.len() - ); - let qos = self.qos_owners(); - let buffer = self.buffer_id(); - let pool = self.free_list.id(); - while let Some(front) = self.active.front_mut() { + let context_update = front.context_update; + self.note_retirement(ordinal, context_update); + let front = self.active.front_mut().ok_or(EIO)?; if !front.flist_completed { host.complete_render( (!front.qos_completed).then_some(qos), @@ -1657,73 +1129,74 @@ impl Pair { front.generation, buffer, )?; - front.flist_completed = true; front.qos_completed = true; + front.flist_completed = true; } - let ordinal = front.ticket.ordinal; - let completion = front.packet.completion.clone(); - completion.classify(recovery::Class::Spared); - defer(Deferred::Retired(completion, Err(ENODATA)))?; + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok(timestamps), + ))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; + return Ok(true); } - Ok(true) } - /// A front group a restart interrupted after its tiler started, which firmware - /// then retired without writing its fragment end, may have run partly. It is - /// not replayed (blending or storage writes could apply twice) and fails now, - /// reported to its owner, instead of waiting out the render timeout. Returns - /// None when the front is not such a group. - fn settle_interrupted_front( + /// The front group was retired and unlinked by firmware without executing + /// (job list empty, stamps cover it, no recovery, fragment end never + /// written). Nothing of this group reached its attachments, so the same + /// command is published again instead of failing the owner's VM. The + /// proof is the strict timeout witness, taken twice ABANDON_GRACE_NS apart, + /// and only for a lone group whose completion still accepts a replay. + fn replay_abandoned( &mut self, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, - grace: bool, - ) -> Result> { + ) -> Result { use retirement::Memory; - let qids = self.qids(); - let Some(front) = self.active.front() else { - return Ok(None); - }; - if !front.outer_published || front.failed || !front.packet.completion.interrupted() { - return Ok(None); + if self.quarantined || self.active.len() != 1 || self.replay.is_some() { + return Ok(false); + } + let front = self.active.front().ok_or(EIO)?; + if !front.outer_published || front.failed { + return Ok(false); } - let ordinal = front.ticket.ordinal; - let ticket = front.ticket; let mut observer = Observer { memory: &self.memory, - qids, - host: &mut *host, + qids: self.qids(), + host, }; core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(ordinal)?; + let snapshot = observer.snapshot(front.ticket.ordinal)?; let stamps = observer.stamps()?; let (_, state) = observer.host.epoch()?; - if snapshot.untouched() - || !matches!( - retirement::abandoned(&ticket, snapshot, stamps, state, false), - Ok(true) - ) - { - return Ok(None); - } + let abandoned = + retirement::abandoned(&front.ticket, snapshot, stamps, state, false)?; let now = crate::g17::now_ns(); let front = self.active.front_mut().ok_or(EIO)?; - if grace { - let Some(since) = front.abandoned_since else { - front.abandoned_since = Some(now); - return Ok(Some(false)); - }; - if now.saturating_sub(since) < ABANDON_GRACE_NS { - return Ok(Some(false)); - } + if !abandoned { + front.abandoned_since = None; + return Ok(false); + } + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(false); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(false); } + if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + pr_info!( + "render slot {} retired without execution (pass {:?}); replaying\n", + self.slot, + snapshot.pass + ); + } + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; if !front.flist_completed { - let qos = self.qos_owners(); - let buffer = self.buffer_id(); - let pool = self.free_list.id(); - let front = self.active.front_mut().ok_or(EIO)?; host.complete_render( (!front.qos_completed).then_some(qos), pool, @@ -1733,32 +1206,39 @@ impl Pair { front.flist_completed = true; front.qos_completed = true; } - pr_warn!( - "G17: render slot {}: group {} was interrupted by a GPU recovery after it started; failing it\n", - self.slot, - ordinal - ); - let packet = self.active.front().ok_or(EIO)?.packet.clone(); - defer(Deferred::Retired(packet.completion.clone(), Err(ECANCELED)))?; + let ordinal = front.ticket.ordinal; + let packet = front.packet.clone(); + if !packet.completion.begin_replay() { + if !packet.completion.spared() { + packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), EIO)); + } + } + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; - Ok(Some(true)) + packet.reset_publication_for_replay(); + self.replay = Some(packet); + Ok(true) } /// The pending replay cannot proceed (closed owner, cancellation or a - /// publication refusal). Its successors cannot run without it and fail with - /// it. ECANCELED leaves the VM usable; any other error is the ordinary - /// device-loss outcome the replay tried to avoid. + /// publication refusal). ECANCELED leaves the VM usable; any other error + /// is the ordinary device-loss outcome the replay tried to avoid. pub(crate) fn fail_replay( &mut self, error: Error, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - self.replay_since = None; - while let Some(packet) = self.replay.pop() { - defer(Deferred::Retired(packet.completion.clone(), Err(error)))?; - } - Ok(()) + let Some(packet) = self.replay.take() else { + return Ok(()); + }; + defer(Deferred::Retired(packet.completion.clone(), Err(error))) } /// A closed preparation epoch invalidates only this unpublished descriptor. @@ -1786,44 +1266,7 @@ impl Pair { let counters = self.free_list.counters()?; let stamps = host.stamps(self.qids())?; let (_, state) = host.epoch()?; - Ok(retirement::drained(self.ordinal, stamps, counters, state) - || (state == 0 && self.executed_unstamped() && self.context_killed())) - } - - /// The firmware acknowledged a kill of this pair's context (scheduler page): it no longer - /// walks that context's work list, so the commands it never unlinked can be released. - fn context_killed(&self) -> bool { - crate::g17::queue::compute::exit::State::read(&self.memory.context) - .is_ok_and(|state| state.killed != 0) - } - - /// The pair's context may be killed although its commands were not retired: they all ran - /// (`executed_unstamped`), so the kill leaves nothing queued for the firmware to run. - /// Its memory stays until the kill is acknowledged. - pub(crate) fn executed_for_kill(&self, context: &Arc) -> bool { - Arc::ptr_eq(context, &self.memory.context) && self.executed_unstamped() - } - - /// Every command of this quarantined pair ran, its ring entries were consumed and its - /// free-list blocks returned, yet the firmware never stores the stamp of the last one. - /// On Neo this is the render work of a context whose compute command failed: it runs - /// about a second after the failure, but its stamp never arrives and it stays linked in - /// the firmware's job list, so neither the stamp proof nor the abandoned-group proof - /// ever holds and the pair would stay quarantined, keeping its context, until the - /// processors stop. - fn executed_unstamped(&self) -> bool { - self.quarantined - // Only for a closed owner: a live owner would go on publishing into a context - // whose work list still links the unstamped node. - && self.owner.is_none() - && self.prepared.is_none() - && self.active.len() != 0 - && self.free_list.counters().is_ok_and(|counters| counters[0] == counters[1]) - && self.active.iter().all(|active| { - active.outer_published - && observation::snapshot(&self.memory, active.ticket.ordinal) - .is_ok_and(retirement::Snapshot::executed_and_consumed) - }) + Ok(retirement::drained(self.ordinal, stamps, counters, state)) } /// Account one witnessed tracker, preserving all completion ownership. @@ -1837,10 +1280,6 @@ impl Pair { let qids = self.qids(); let qos = self.qos_owners(); let buffer = self.buffer_id(); - // Commands the firmware never unlinked are released only after it acknowledged the - // kill of their context, on every path (see `firmware_drained`). - let unstamped = self.executed_unstamped(); - let killed = unstamped && self.context_killed(); let Some(active) = self.active.iter_mut().find(|a| !a.flist_completed) else { return Ok(false); }; @@ -1851,12 +1290,6 @@ impl Pair { }; let result = match retirement::poll(&mut active.ticket, &mut observer)? { retirement::Poll::Complete(ticks) => Ok(ticks), - // Its commands ran without a stamp (see `executed_unstamped`): they already failed - // with the quarantine, and the client sees them cancelled. - retirement::Poll::Pending | retirement::Poll::Retry if killed => { - crate::g17::trace::teardown(2, u64::from(self.slot), active.ticket.ordinal, 0); - Err(ECANCELED) - } retirement::Poll::Pending | retirement::Poll::Retry => { core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); let snapshot = observer.snapshot(active.ticket.ordinal)?; @@ -1924,49 +1357,22 @@ impl Pair { if !self.active.iter().any(|a| Arc::ptr_eq(&a.packet, packet)) { return Ok(false); } - // Groups the firmware dropped unexecuted are published again, with their successors. - if self.replay_abandoned(host, defer, false)? { - return Ok(true); - } - // Fallback when a live or exhausted group prevents an ordered replay: a spared front group - // the firmware retired without executing during another VM's recovery is retired on its - // own even with successors queued, so the pair is not quarantined into a sticky EIO. It - // fails with ECANCELED, which spared work never reports as device loss; consecutive - // abandoned spared successors are retired the same way. - let front_spared = self - .active - .front() - .is_some_and(|a| Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared()); - if !self.quarantined && (self.active.len() == 1 || front_spared) { - let mut retired = false; - while let Some(active) = self.active.front() { - let spared = active.packet.completion.spared(); - if retired && !spared { - break; - } - let mut observer = Observer { - memory: &self.memory, - qids: self.qids(), - host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(active.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - // A torn sample is no proof, as in replay_abandoned(): the group stays retained - // (quarantined with the pair if nothing was retired) instead of the error - // stopping the runtime for every client. - if !active.outer_published - || !matches!( - retirement::abandoned(&active.ticket, snapshot, stamps, state, false), - Ok(true) - ) - { - break; - } + if !self.quarantined && self.active.len() == 1 { + let active = self.active.front().ok_or(EIO)?; + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if active.outer_published + && retirement::abandoned(&active.ticket, snapshot, stamps, state, false)? + { let qos = self.qos_owners(); let buffer = self.buffer_id(); - let left = self.active.len() - 1; let active = self.active.front_mut().ok_or(EIO)?; if !active.flist_completed { host.complete_render( @@ -1978,30 +1384,19 @@ impl Pair { active.flist_completed = true; active.qos_completed = true; } - if !active.failed && !spared { + if !active.failed && !active.packet.completion.spared() { active.packet.completion.status().record(EIO); if self.terminal.is_none() { self.terminal = Some((active.packet.completion.status().clone(), error)); } } - pr_warn!( - "G17: render retired unexecuted after a recovery (ordinal {}, {} groups left)\n", - active.ticket.ordinal, - left - ); defer(Deferred::Retired( active.packet.completion.clone(), - Err(if spared { ECANCELED } else { EIO }), + Err(EIO), ))?; let ordinal = active.ticket.ordinal; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; - retired = true; - if !spared { - break; - } - } - if retired { return Ok(true); } } @@ -2029,8 +1424,7 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; } - self.replay_since = None; - while let Some(packet) = self.replay.pop() { + if let Some(packet) = self.replay.take() { defer(Deferred::Drained(packet.completion.clone(), Err(EIO)))?; } if let Some(previous) = self.previous.as_mut() { @@ -2184,87 +1578,6 @@ impl Pair { } } } - /// The scheduler page these queues last ran a command for, when the next - /// publication for `context` changes the queues' owner. - /// The steady state in which a command may publish without the device - /// mutex: the graph is installed and registered at `priority`, it last ran - /// `context`'s scheduler, and nothing is replayed, settled, aliased or - /// switched on it. - pub(crate) fn direct_ready(&self, context: &Context, priority: u8) -> bool { - self.graph_started - && self.installed == [true; 2] - && self.registered == Some(priority) - && priority < 8 - && self.outer_started & (1u8 << priority) != 0 - && self.executed == context.scheduler_va() - && self.switch.is_none() - && self.owner.is_some() - && !self.quarantined - && self.terminal.is_none() - && self.pending_aliases.is_none() - && self.kick_aliases.is_none() - && self.prepared.is_none() - && self.replay.len() == 0 - && self.displaced.is_none() - && self.active.room() - } - /// The render prefix of `packet` on this pair, as the registry collects it - /// for an owner whose only pair this is. - pub(crate) fn own_dependencies( - &self, - owner: u64, - packet: &Arc, - ) -> Result { - let mut dependencies = RenderDependencies::EMPTY; - let mut frontier = crate::g17::dependency::Frontier::new(); - self.collect_dependencies( - owner, - packet.completion.status(), - packet.order.wait_through[0], - &mut frontier, - false, - )?; - for dependency in frontier.as_slice() { - *dependencies - .render - .get_mut(dependencies.render_count) - .ok_or(EOVERFLOW)? = *dependency; - dependencies.render_count += 1; - } - if dependencies.render_count != 0 - && matches!(&packet.command, Validated::Render { pass, .. } if pass.fragment_barriers) - { - let mut tiling = crate::g17::dependency::Frontier::new(); - self.collect_tiling_dependencies( - owner, - packet.completion.status(), - packet.order.wait_through[0], - &mut tiling, - )?; - for dependency in tiling.as_slice() { - *dependencies - .tiling - .get_mut(dependencies.tiling_count) - .ok_or(EOVERFLOW)? = *dependency; - dependencies.tiling_count += 1; - } - } - Ok(dependencies) - } - pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { - (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) - } - /// The previous scheduler page of an owner change whose first command has - /// not left the pair yet. - pub(crate) fn switch_in_flight(&self) -> Option { - let (from, first) = self.switch?; - (self.replay.len() != 0 - || self - .active - .iter() - .any(|active| active.ticket.ordinal == first)) - .then_some(from) - } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { return self.displaced.take(); @@ -2308,10 +1621,6 @@ impl Pair { self.owner = None; } #[inline(never)] - #[expect( - clippy::too_many_arguments, - reason = "Binding keeps the render owners and first packet explicit." - )] pub(crate) fn prepare_bind( &mut self, alloc: &Allocator<'_>, @@ -2345,7 +1654,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = target::Layout::new(pass, self.clusters)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, @@ -2361,7 +1670,6 @@ impl Pair { pm_generation, self.ordinal, first.completion.work_state()?.word(), - self.variant, )?; (Some(memory), Some(persistent), Some(aliases)) }; @@ -2428,9 +1736,6 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; - if foreign { - self.graph_started = false; - } self.context_update = true; if foreign { self.cursors = [0; 2]; @@ -2439,45 +1744,3 @@ impl Pair { Ok(()) } } - -/// Whether `masks`, the stamp notifications collected for a pair whose tiling and fragment -/// queues are `qids`, can mean a finished pass: the fragment queue's bit. A fragment kick starts -/// only after its fragment barrier saw the tiling stamp, and retirement still checks both stamps -/// in memory, so the tiling notification is not needed to retire. -pub(crate) fn completion_noted(qids: [u8; 2], masks: [u64; 2]) -> bool { - let fragment = usize::from(qids[1]); - masks[fragment / 64] & (1u64 << (fragment % 64)) != 0 -} - -#[kunit_tests(asahi_g17_render_completion_noted)] -mod completion_noted_tests { - use super::completion_noted; - - fn bit(qid: u8) -> [u64; 2] { - let mut masks = [0; 2]; - masks[usize::from(qid) / 64] |= 1u64 << (qid % 64); - masks - } - - #[test] - fn fragment_notification_alone_selects() { - assert!(completion_noted([3, 4], bit(4))); - assert!(completion_noted([70, 100], bit(100))); - } - - #[test] - fn tiling_notification_alone_does_not_select() { - assert!(!completion_noted([3, 4], bit(3))); - assert!(!completion_noted([70, 100], bit(70))); - assert!(!completion_noted([3, 4], [0; 2])); - } - - #[test] - fn other_queues_do_not_select() { - assert!(!completion_noted([3, 4], bit(5))); - assert!(!completion_noted([3, 68], bit(4))); - let mut both = bit(3); - both[1] |= bit(68)[1]; - assert!(completion_noted([3, 68], both)); - } -} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 90a11ca977684..85319585b3d13 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,9 +31,6 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; -/// The lead command of a stage (InitBM before tiling), after the stage's configuration record -/// in its slot ([`crate::g17::fw::Variant::lead_commands`]). -pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); @@ -97,8 +94,6 @@ pub(super) struct Graph { pub(super) config: PooledObject, pub(super) support: KernelObject, pub(super) owners: [Option; RECORDS], - /// Host words of the pointer blocks in this firmware build. - pub(super) pointers: fw::variant::PointerLayout, } impl Graph { pub(super) fn new( @@ -106,7 +101,6 @@ impl Graph { pool: &Arc, context: &Context, qids: [u8; 2], - pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; let descriptors = pool.allocate( @@ -160,7 +154,7 @@ impl Graph { }), )?; queues.initialize::(POINTERS[stage], |view| { - *view = queue::PointerBlock::new(&pointers)?; + *view = queue::PointerBlock::new(); Ok(()) })?; } @@ -172,7 +166,6 @@ impl Graph { config, support, owners: [None; RECORDS], - pointers, }) } pub(super) fn offset(stage: usize, ordinal: u64) -> usize { @@ -195,13 +188,11 @@ impl Graph { self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 } pub(super) fn publish_recovery_targets(&self) -> Result { - let Some(target) = self.pointers.recovery_target else { - return Ok(()); - }; for at in POINTERS { - let producer = self.queues.word(at + self.pointers.producer)? + let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? .load(Ordering::Relaxed); - self.queues.word(at + target)?.store(producer, Ordering::Relaxed); + self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? + .store(producer, Ordering::Relaxed); } Ok(()) } @@ -217,7 +208,7 @@ impl Graph { values[stage] = Cursors { done: self.queues.word(at)?.load(Ordering::Relaxed), read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), - write: self.queues.word(at + self.pointers.producer)?.load(Ordering::Relaxed), + write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), }; } Ok(values) @@ -319,36 +310,24 @@ impl Generation { ) .map_err(Into::into) } - /// With `viewport` false the deflake buffer has no room for the viewport words - /// ([`abi::Deflake::viewport`]). - pub(super) fn prepare( - &mut self, - ordinal: u64, - pass: &abi::RenderPass, - viewport: bool, - ) -> Result { + pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } - let viewport = match viewport { - true => Some(storage::Viewport::new(pass.width, pass.height)?), - false => None, - }; + let viewport = storage::Viewport::new(pass.width, pass.height)?; self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] .fill(0); - if let Some(viewport) = viewport { - let destination = self - .state - .pointer(storage::DEFLAKE + 0x900, size_of::())? - .cast::(); - if destination.align_offset(align_of::()) != 0 { - return Err(EINVAL); - } - // SAFETY: The complete typed integer view is in the retained CPU mapping - // and naturally aligned. This command generation is exclusively owned - // and unpublished; no prior command can access it until reassigned. - unsafe { viewport.apply(&mut *destination) }; + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; self.fragment.bytes_mut().fill(0); self.ordinal = Some(ordinal); Ok(()) @@ -386,10 +365,6 @@ pub(super) struct Memory { } impl Memory { #[inline(never)] - #[expect( - clippy::too_many_arguments, - reason = "Render memory construction keeps each allocation and firmware input explicit." - )] pub(super) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -401,11 +376,10 @@ impl Memory { pm_generation: u64, ordinal: u64, word: u32, - variant: &'static fw::Variant, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); - let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; + let mut graph = Graph::new(alloc, pool, &context, qids)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; let state = Generation::allocate_state(alloc, vm)?; @@ -528,44 +502,12 @@ impl Memory { } const _: () = { - // Every unretired render of a pair holds its own command record. - assert!(DEPTH <= RECORDS); assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); - assert!(size_of::() <= LEAD); - assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; impl Memory { - /// Requests the tiling descriptor slot and both queue configuration records of - /// `ordinal` for writing. The firmware accessed them for the command that used them - /// last, so the host no longer holds their lines; requesting them before the rest of - /// the preparation overlaps those transfers with it instead of stalling the writes one - /// line at a time. - pub(super) fn prefetch_descriptors(&self, ordinal: u64) { - let record = ordinal as usize % RECORDS; - let config = size_of::(); - let ranges = [ - (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), - (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), - (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), - ]; - for (object, offset, size) in ranges { - let Ok(base) = object.pointer(offset, size) else { - continue; - }; - for line in (0..size).step_by(64) { - // SAFETY: A prefetch hint inside the retained mapping; it has no - // architectural effect on memory. - unsafe { - core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(line), - options(nostack, preserves_flags, readonly)); - } - } - } - } - pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, @@ -600,10 +542,21 @@ impl Memory { (&mut *tiling).write(args)?; let fragment = &mut *fragment; fragment.write(args)?; - // Kick entries bind the arrays embedded in this descriptor - // (register_bindings); the client register page is never read. - for array in &fragment.descriptor.arrays { - array.entries.get(..(array.size & 0xffff) as usize).ok_or(EINVAL)?; + self.registers.bytes_mut().fill(0); + for (index, array) in fragment.descriptor.arrays.iter().enumerate() { + for (entry, value) in array + .entries + .get(..(array.size & 0xffff) as usize) + .ok_or(EINVAL)? + .iter() + .enumerate() + { + self.registers.write( + index * size_of::() + + entry * size_of::(), + *value, + )?; + } } fragment.set_mcache( args.fragment.descriptor_va, @@ -645,33 +598,6 @@ impl Memory { } Ok(()) } - - /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the - /// tiling descriptor when `init_manager`. Returns which stages have one. - pub(super) fn write_leads(&mut self, ordinal: u64, init_manager: bool) -> Result<[bool; 2]> { - if !self.owns(ordinal) { - return Err(EIO); - } - let ta = self - .graph - .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? - .cast::(); - // SAFETY: The view was checked during preparation and the ordinal still owns it; only - // integer fields are read. - let ta = unsafe { &*ta }; - let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; - if init_manager { - let init = queue::InitBufferManager::new( - ta.context_id, - ta.pb_slot, - ta.pb_va, - ta.stamp_value, - ); - self.graph.config.write(CONFIG[0] + slot, init)?; - } - Ok([init_manager, false]) - } } impl Memory { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 81d4306eda28c..39941de815e07 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -25,19 +25,12 @@ pub(crate) struct ParkedPair { slot: u8, ordinal: u64, clusters: u32, - deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], - variant: &'static crate::g17::fw::Variant, installed: [bool; 2], recovery_generation: u64, registered: Option, outer_started: u8, previous: Option, - /// Scheduler page these queues last ran a command for. A pair parked - /// without a consumed release (an unused warm pair taken for another - /// owner) keeps it, so its next owner change still waits while that - /// scheduler is released. - executed: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -55,20 +48,11 @@ pub(crate) struct Prepared { } impl Pair { - /// The current graph never entered command publication. Physical identities - /// may have older users; parking retains their rings and retirement owners. - pub(crate) fn can_yield_unused(&self) -> Result { - Ok(!self.graph_started && self.can_park_backing(self.context())?) - } - /// The registry also checks the shared free-list release and scheduler-release witness. /// Drain displaced owners before this preflight; they have their own logical releases. pub(crate) fn can_park(&self, context: &Arc) -> Result { - Ok(self.owner.is_none() && self.can_park_backing(context)?) - } - - fn can_park_backing(&self, context: &Arc) -> Result { Ok(Arc::ptr_eq(context, self.context()) + && self.owner.is_none() && self .previous .as_ref() @@ -92,15 +76,12 @@ impl Pair { slot: self.slot, ordinal: self.ordinal, clusters: self.clusters, - deflake: self.deflake, descriptor_flags: self.descriptor_flags, - variant: self.variant, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, outer_started: self.outer_started, previous: self.previous, - executed: self.executed, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -115,22 +96,11 @@ impl Pair { } impl ParkedPair { - /// The release of `scheduler` has been consumed: a later owner change no - /// longer has to wait for it. - pub(crate) fn release_consumed(&mut self, scheduler: u64) { - if self.executed == scheduler { - self.executed = 0; - } - } pub(crate) fn qids(&self) -> [u8; 2] { self.ids.map(kick::Id::qid) } #[inline(never)] - #[expect( - clippy::too_many_arguments, - reason = "Preparation forwards the render binding inputs without changing their ownership." - )] pub(crate) fn prepare( &self, alloc: &Allocator<'_>, @@ -155,7 +125,7 @@ impl ParkedPair { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let layout = target::Layout::new(pass, self.clusters)?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, @@ -171,7 +141,6 @@ impl ParkedPair { pm_generation, self.ordinal, word, - self.variant, )?; Ok(Prepared { memory: Some(memory), @@ -221,7 +190,6 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { - graph_started: false, kick_aliases: None, memory, kicks, @@ -234,9 +202,7 @@ impl ParkedPair { ordinal: self.ordinal, initial_packet: next.initial_packet, clusters: self.clusters, - deflake: self.deflake, descriptor_flags: self.descriptor_flags, - variant: self.variant, installed: self.installed, configs: [None; 2], cursors: [0; 2], @@ -247,12 +213,8 @@ impl ParkedPair { terminal: None, previous: self.previous.take(), displaced: None, - replay: Tracker::new(), - replay_since: None, + replay: None, context_update: true, - executed: self.executed, - switch: None, - published_completions: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index d65c40c7cfade..0fc9c1d9a15dc 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -19,7 +19,7 @@ fn identity(mut config: QueueConfig) -> QueueConfig { config.qos_update = 0; config } -pub(in crate::g17::queue) fn required(previous: Option, config: QueueConfig) -> bool { +pub(super) fn required(previous: Option, config: QueueConfig) -> bool { config.install != 0 || config.context_update != 0 || config.completion_seed_valid != 0 @@ -28,9 +28,9 @@ pub(in crate::g17::queue) fn required(previous: Option, config: Que } pub(super) struct Lane<'a> { - items: [&'a AtomicU64; 4], + items: [&'a AtomicU64; 3], producer: &'a AtomicU32, - values: [u64; 4], + values: [u64; 3], config: QueueConfig, config_destination: *mut u64, config_present: bool, @@ -41,10 +41,6 @@ pub(super) struct Lane<'a> { pub(super) window: Window, } impl<'a> Lane<'a> { - #[expect( - clippy::too_many_arguments, - reason = "The publication lane describes independent fields of the firmware wire record." - )] pub(super) fn new( graph: &'a memory::Graph, stage: usize, @@ -53,9 +49,8 @@ impl<'a> Lane<'a> { config: QueueConfig, present: bool, timestamp: KickTimestamp, - lead: bool, ) -> Result { - let count = 2 + u32::from(present) + u32::from(lead); + let count = 2 + u32::from(present); let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -77,30 +72,21 @@ impl<'a> Lane<'a> { graph.queues.dword(index(0))?, graph.queues.dword(index(1))?, graph.queues.dword(index(2))?, - graph.queues.dword(index(3))?, ]; let announce_va = graph.queues.gpu_va() + announce_offset as u64; let descriptor = graph.descriptor_va(stage, ordinal); - // Item order: [lead command], descriptor, [configuration], announcement. - let mut values = [0; 4]; - let mut n = 0; - let mut push = |value: u64| { - values[n] = value; - n += 1; + let values = if present { + [ + descriptor, + graph.config.gpu_va() + config_offset as u64, + announce_va, + ] + } else { + [descriptor, announce_va, 0] }; - if lead { - push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); - } - push(descriptor); - if present { - push(graph.config.gpu_va() + config_offset as u64); - } - push(announce_va); Ok(Self { items, - producer: graph - .queues - .word(memory::POINTERS[stage] + graph.pointers.producer)?, + producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, values, config, config_destination, @@ -130,7 +116,7 @@ impl<'a> Lane<'a> { if !self.config_present { return; } - let source = core::ptr::from_ref(&self.config).cast::(); + let source = (&self.config as *const QueueConfig).cast::(); for index in 0..size_of::() / 8 { // SAFETY: QueueConfig is packed integer storage with no padding; // every source qword is read unaligned. The destination is a diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 4377bbfe3e311..d0a2d566bdc8d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -122,17 +122,6 @@ impl Snapshot { fn executed(self) -> bool { self.pass[0] != 0 && self.pass[1] != 0 } - - /// Neither the tiler start nor the fragment end was written: nothing ran. - pub(crate) fn untouched(self) -> bool { - self.pass == [0, 0] - } - - /// The command ran (tiler start and fragment end written) and the firmware consumed both - /// queues' ring entries. - pub(crate) fn executed_and_consumed(self) -> bool { - self.executed() && self.cursors.iter().all(|cursors| cursors.done == cursors.write) - } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -155,85 +144,6 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result bool) -> StampWait { - if now.saturating_sub(since) < STAMP_FALLBACK_NS { - StampWait::Waiting - } else if since == 0 || drained() { - StampWait::Expired - } else { - StampWait::Restart(now) - } -} - -static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); - -/// Counts a render group that retired on its pass record because its completion stamps never -/// covered it; logs the first and every 64th. -pub(crate) fn note_stampless(slot: u16, ordinal: u64) { - let count = STAMPLESS.fetch_add(1, Ordering::Relaxed) + 1; - if count == 1 || count % 64 == 0 { - pr_info!( - "G17: lifetime total {} render groups retired on their pass record after {} ms without completion stamps (latest: slot {} ordinal {})\n", - count, - STAMP_FALLBACK_NS / 1_000_000, - slot, - ordinal - ); - } -} - -/// Retire an executed, consumed front group without its stamps, once [`STAMP_FALLBACK_NS`] -/// has passed. The caller decides that bound; this re-checks execution and consumption. Unlike -/// the stamped path, it has no post-maintenance evidence; see [`STAMP_FALLBACK_NS`]. -pub(crate) fn poll_stampless(ticket: &mut Ticket, memory: &mut impl Memory) -> Result { - fence(Ordering::Acquire); - let snapshot = memory.snapshot(ticket.ordinal)?; - let Ok(observation) = observe(ticket, snapshot) else { - return Ok(Poll::Retry); - }; - if !observation.complete { - return Ok(Poll::Pending); - } - let [start, end] = memory.pass(ticket.ordinal)?; - Ok(Poll::Complete([start, start, start, end])) -} - /// Outstanding publications span less than half the 24-bit stamp sequence. /// A zero stamp is valid when that sequence wraps. pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool { @@ -293,12 +203,8 @@ pub(crate) fn poll_observed( memory: &mut impl Memory, snapshot: Snapshot, ) -> Result { - match observe(ticket, snapshot) { - Ok(observation) => complete_poll(ticket, memory, snapshot, observation), - // The borrowed snapshot can straddle a firmware cursor update like any other sample: - // take the ordinary poll's fresh snapshot and reread instead of failing the client. - Err(_) => poll(ticket, memory), - } + let observation = observe(ticket, snapshot)?; + complete_poll(ticket, memory, snapshot, observation) } fn complete_poll( @@ -440,94 +346,3 @@ impl Tracker { .filter_map(Option::as_mut) } } - -#[kunit_tests(asahi_g17_render_retirement)] -mod executed_tests { - use super::*; - - fn snapshot(done: u32, write: u32, pass: [u64; 2]) -> Snapshot { - Snapshot { - cursors: [Cursors { done, read: done, write }; 2], - job_list_empty: false, - pass, - } - } - - #[test] - fn executed_and_consumed_needs_both_passes_and_consumed_rings() { - assert!(snapshot(3, 3, [1, 2]).executed_and_consumed()); - assert!(!snapshot(3, 3, [0, 0]).executed_and_consumed()); - assert!(!snapshot(3, 3, [1, 0]).executed_and_consumed()); - assert!(!snapshot(3, 3, [0, 2]).executed_and_consumed()); - assert!(!snapshot(2, 3, [1, 2]).executed_and_consumed()); - } - - /// The firmware state of a stalled pair as read on hardware: the front group executed and - /// its two ring entries per queue were consumed, but both stamps hold the previous kick. - struct Stalled { - snapshot: Snapshot, - stamps: [u32; 2], - } - impl Memory for Stalled { - fn snapshot(&mut self, _ordinal: u64) -> Result { - Ok(self.snapshot) - } - fn stamps(&mut self) -> Result<[u32; 2]> { - Ok(self.stamps) - } - fn pass(&mut self, _ordinal: u64) -> Result<[u64; 2]> { - Ok(self.snapshot.pass) - } - } - - fn stalled_ticket() -> Ticket { - let window = Window::new(1097, 2, 1280).unwrap(); - let previous = Cursors { done: 1097, read: 1097, write: 1097 }; - Ticket { ordinal: 406, windows: [window; 2], previous: [previous; 2] } - } - - fn stalled(pass: [u64; 2]) -> Stalled { - let cursors = Cursors { done: 1099, read: 1099, write: 1099 }; - Stalled { - snapshot: Snapshot { cursors: [cursors; 2], job_list_empty: false, pass }, - stamps: [406 << 8; 2], - } - } - - #[test] - fn lagging_stamps_keep_an_executed_group_pending() { - let mut ticket = stalled_ticket(); - let mut memory = stalled([10, 20]); - assert!(matches!(poll(&mut ticket, &mut memory), Ok(Poll::Pending))); - } - - #[test] - fn stampless_poll_retires_an_executed_consumed_group() { - let mut ticket = stalled_ticket(); - let mut memory = stalled([10, 20]); - assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Complete([10, 10, 10, 20])))); - } - - #[test] - fn stampless_poll_waits_for_the_fragment_end() { - let mut ticket = stalled_ticket(); - let mut memory = stalled([10, 0]); - assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending))); - } - - #[test] - fn stamps_wait_for_the_rest_of_the_ioctl() { - let since = 1_000_000; - let late = since + STAMP_FALLBACK_NS; - // The node is asked only once the bound passed. - assert_eq!(stamp_wait(since, late - 1, || panic!("node read early")), StampWait::Waiting); - // Other kicks of the group's ioctl still run: the bound restarts. - assert_eq!(stamp_wait(since, late, || false), StampWait::Restart(late)); - assert_eq!(stamp_wait(late, late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); - // A drained node with lagging stamps is the stall the bound exists for. - assert_eq!(stamp_wait(since, late, || true), StampWait::Expired); - // The watchdog's last resort does not wait for the node. - assert_eq!(stamp_wait(0, STAMP_FALLBACK_NS, || panic!("node read for an expired wait")), - StampWait::Expired); - } -} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs index da90488bfedec..4db471f254ec6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -36,11 +36,10 @@ impl Pair { } *block = pointer.cast(); } - let layout = self.memory.graph.pointers; for block in blocks { // SAFETY: Both complete pointer blocks were checked above. This fresh // graph is unpublished; only the live atomic cursors are accessed. - unsafe { &*block }.rebase(&layout, 0)?; + unsafe { &*block }.rebase(0); } fence(Ordering::SeqCst); self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index ca8da8b03545f..ba368341d84d2 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -9,13 +9,11 @@ mod attribution; mod slot; -mod safety; -pub(crate) use safety::{note_retired, since_retired}; pub(crate) use attribution::Sources; use attribution::Attribution; -pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { - slot::key(slot, slot_keys?).ok() +pub(super) fn slot_key(slot: usize) -> Option { + slot::key(slot).ok() } use super::status::VmStatus; @@ -25,34 +23,20 @@ const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; const MAX_GUILTY_VMS: usize = 8; -/// One-millisecond ticks per handshake phase. The firmware finishes a restart only after the -/// work it interrupted has drained; with several clients running long ray-traversal jobs that -/// takes seconds, and giving up loses every client's device. Ten seconds matches the longest -/// execution allowance (`timeout::POLL_MS` plus renewals). -const RECOVERY_POLLS: usize = 10_000; +const RECOVERY_POLLS: usize = 500; -/// Records in the firmware recovery table. -pub(super) const RECOVERY_RECORDS: usize = 256; - -/// Per-QID classifications from the firmware recovery table. +/// Per-QID classifications from the complete firmware recovery table. /// Multiple records for a queue contribute all of their classification bits. #[derive(Default)] pub(super) struct Entries { guilty: u128, cancelled: u128, - /// Valid records, each naming one command that was in flight at the restart. - pub(super) listed: u32, - /// Listed commands found by their exact stamp, and by the oldest-work fallback. - pub(super) exact: u32, - pub(super) fallback: u32, } impl Entries { - /// Returns false at the first invalid record: the table is a packed list and - /// records after it are left over from an earlier restart. - pub(super) fn include(&mut self, flags: u32) -> bool { + pub(super) fn include(&mut self, flags: u32) { if flags & 1 == 0 { - return false; + return; } let qid = (flags >> 4) & 0x7f; let bit = 1u128 << qid; @@ -62,22 +46,9 @@ impl Entries { if flags & u32::from(CANCELLED) != 0 { self.cancelled |= bit; } - self.listed += 1; - true } } -/// How a listed recovery record was matched to retained work. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(super) enum Interrupted { - /// No retained command of the queue. - None, - /// The command whose completion stamp equals the record's. - Exact, - /// No stamp matched; the queue's oldest unretired command was marked instead. - Oldest, -} - /// How an error completion affects its VM's sticky status and other jobs. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Class { @@ -99,12 +70,6 @@ pub(super) trait Queues { fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); /// Classify every in-flight completion; return QIDs of render slots with spared work. fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; - /// Mark the command a recovery record lists as interrupted. It may have - /// partly executed, so it must never report success; see - /// `Completion::note_interrupted`. - fn note_interrupted(&mut self, qid: u8, stamp: u32) -> Interrupted; - /// Optional diagnostics only; absence cannot alter attribution or release. - fn report_blamed_render(&self, _qid: u8) {} fn settle_blamed_render(&mut self, qid: u8, error: Error); fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; @@ -119,9 +84,7 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - /// One recovery record's flags and completion stamp, read while firmware - /// is halted in state 3. - fn entry(&self, index: usize) -> Result<(u32, u32)>; + fn entries(&self) -> Result; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -136,8 +99,6 @@ struct Request { pub(super) struct State { accepted: u64, pending: Option, - timeouts: safety::Storm, - restarts: safety::Storm, } /// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. @@ -151,14 +112,8 @@ impl State { Self { accepted: 0, pending: None, - timeouts: safety::Storm::new(), - restarts: safety::Storm::new(), } } - pub(super) fn timeout_limit(&mut self, now: u64) -> bool { - self.timeouts.note_unless_progress(now, safety::retired()) - } - pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted } @@ -230,8 +185,6 @@ pub(super) fn service( } } if !ready { - pr_warn!("G17: recovery gen {} did not reach state 1 (state {:?})\n", - request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // Recheck ownership after the off-CPU waits, before any handshake mutation. @@ -243,15 +196,6 @@ pub(super) fn service( host.clear_timestamps()?; } let mut sources = host.fault_sources()?; - sources.report(request.generation, super::queue::compute::retirement::stampless_total()); - if let Some(qid) = request.blamed { - host.report_blamed_render(qid); - } - // The keys, monitor and pass-start evidence must describe the same halted - // instant. Firmware may update pass timestamps as soon as state 3 is - // acknowledged; sampling them during classification can blame a bystander. - let attribution = sources.attribute(request.blamed, - |qid, stamp| host.render_pass_started(qid, stamp)); let target_error = host.publish_recovery_targets(); host.transition(1, 2)?; let mut recovered = false; @@ -266,41 +210,19 @@ pub(super) fn service( host.wait_tick(); } if !recovered { - pr_warn!("G17: recovery gen {} did not reach state 3 (state {:?})\n", - request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // A failed table read leaves only the explicit blamed QID as evidence of guilt. - // Every listed command was in flight when firmware stopped. Firmware may - // still retire it with stamps and records that look like success, so it is - // marked before firmware resumes and never reports success. - let entries = read_entries(host); + let entries = host.entries().unwrap_or_default(); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; - let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); - let (cancelled, blame) = classify(host, entries, request.blamed, attribution); - if listed != 0 { - pr_warn!( - "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", - listed, - exact, - fallback - ); - } + let cancelled = classify(host, entries, request.blamed, &mut sources); if let Some(qid) = request.blamed { - pr_warn!( - "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", - sources.reason, - qid, - if blame { "failing its oldest render" } else { "attribution cleared it; spared" } - ); - if blame { - host.settle_blamed_render(qid, EIO); - } + host.settle_blamed_render(qid, EIO); } if cancelled != 0 { host.settle_cancelled_renders(cancelled); @@ -310,44 +232,21 @@ pub(super) fn service( Ok(ServiceResult { scan_error, target_error }) } -/// Walks the packed recovery table up to its first invalid record and marks -/// every listed command interrupted. A failed read keeps what was read so far. -fn read_entries(host: &mut H) -> Entries { - let mut entries = Entries::default(); - for index in 0..RECOVERY_RECORDS { - let Ok((flags, stamp)) = host.entry(index) else { - break; - }; - if !entries.include(flags) { - break; - } - match host.note_interrupted(((flags >> 4) & 0x7f) as u8, stamp) { - Interrupted::Exact => entries.exact += 1, - Interrupted::Oldest => entries.fallback += 1, - Interrupted::None => {} - } - } - entries -} - /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. -/// Also returns whether the firmware's blamed QID is still guilty after attribution: a blamed -/// render that had not started while others had (or with no culprit at all) is innocent, and its -/// work is spared like that of every other innocent VM instead of failing with EIO. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, attribution: Attribution) -> (u128, bool) { +fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> u128 { let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); let cancelled = entries.cancelled; + let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); match attribution { Attribution::Firmware => {} Attribution::Queues(qids) => guilty_qids = qids, Attribution::NoCulprit => guilty_qids = 0, } let no_culprit = attribution == Attribution::NoCulprit; - let blame = blamed.is_some_and(|qid| guilty_qids & (1u128 << qid) != 0); if guilty_qids == 0 && !no_culprit { - return (0, blame); + return 0; } let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); let mut count = 0; @@ -383,7 +282,7 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, att } } if unresolved || (count == 0 && collateral == 0 && !no_culprit) { - return (0, blame); + return 0; } let spared = queues.classify_work(&|status| { guilty[..count] @@ -396,103 +295,5 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, att }) .unwrap_or(Class::Spared) }); - (cancelled & !(guilty_qids & !collateral) & spared, blame) -} - - -#[kernel::macros::kunit_tests(asahi_g17_recovery_snapshot)] -mod tests { - use super::*; - use core::cell::Cell; - - struct Halted { - recovery: State, - state: Cell, - reads: Cell, - known_keys: bool, - blamed: bool, - } - - impl Halted { - fn new(known_keys: bool) -> Self { - let mut recovery = State::new(); - assert!(recovery.request(0, 38).unwrap()); - Self { recovery, state: Cell::new(1), reads: Cell::new(0), - known_keys, blamed: false } - } - } - - impl Memory for Halted { - fn recovery_state(&self) -> Result { Ok(self.state.get()) } - fn host_recovery(&self) -> Result { Ok(0) } - fn fault_sources(&self) -> Result { - assert_eq!(self.state.get(), 1); - let mut sources = Sources::sample(3, |slot| { - Some((if self.known_keys { ((38 + slot as u64) << 40) | 1 } else { 0 }, 0)) - }); - sources.reason = Some(2); - Ok(sources) - } - fn transition(&self, expected: u32, next: u32) -> Result { - assert_eq!(self.state.get(), expected); - self.state.set(next); - Ok(()) - } - fn clear_timestamps(&self) -> Result { panic!("active render scheduler") } - fn entry(&self, _index: usize) -> Result<(u32, u32)> { - assert_eq!(self.state.get(), 3); - Ok((0, 0)) - } - fn wait_tick(&self) { - assert_eq!(self.state.get(), 2); - self.state.set(3); - } - } - - impl Queues for Halted { - fn render_scheduler_active(&self) -> bool { true } - fn render_pass_started(&self, qid: u8, _stamp: u64) -> Option { - // This is the regression oracle: state 0/2/3 is too late. A - // resumed peer may have started after the fault we are attributing. - assert_eq!(self.state.get(), 1); - self.reads.set(self.reads.get() + 1); - Some(qid == 39) - } - fn publish_recovery_targets(&self) -> Option { None } - fn oldest_spared(&self, _qid: u8) -> Option { None } - fn visit_owners(&self, _qid: u8, _visit: &mut dyn FnMut(&Arc)) {} - fn classify_work(&mut self, _class: &dyn Fn(&Arc) -> Class) -> u128 { 0 } - fn note_interrupted(&mut self, _qid: u8, _stamp: u32) -> Interrupted { Interrupted::None } - fn settle_blamed_render(&mut self, qid: u8, _error: Error) { - assert_eq!(qid, 38); - self.blamed = true; - } - fn settle_cancelled_renders(&mut self, _qids: u128) {} - fn rescan_compute(&mut self) -> Result { Ok(()) } - fn release_retired_compute(&mut self) {} - } - - impl Host for Halted { - fn recovery(&self) -> &State { &self.recovery } - fn recovery_mut(&mut self) -> &mut State { &mut self.recovery } - } - - #[test] - fn pass_evidence_is_frozen_before_the_first_acknowledgement() { - let mut host = Halted::new(true); - service(&mut host, || false).unwrap(); - assert_eq!(host.reads.get(), 2); - assert_eq!(host.state.get(), 0); - assert_eq!(host.recovery.generation(), 1); - assert!(!host.blamed); - } - - #[test] - fn unknown_25g83_keys_still_preserve_firmware_blame() { - let mut host = Halted::new(false); - service(&mut host, || false).unwrap(); - assert_eq!(host.reads.get(), 0); - assert!(host.blamed); - assert_eq!(host.state.get(), 0); - } + cancelled & !(guilty_qids & !collateral) & spared } diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs deleted file mode 100644 index d71d3cc81553f..0000000000000 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ /dev/null @@ -1,131 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Fail closed on repeated watchdog/restart traffic, even across client loss. - -use core::sync::atomic::{AtomicU64, Ordering}; - -const WINDOW_NS: u64 = 30_000_000_000; -const LIMIT: u8 = 8; - -/// Commands retired successfully. A watchdog storm means the GPU stopped making progress; -/// timeouts of overloaded clients while other work keeps retiring are not one. -static RETIRED: AtomicU64 = AtomicU64::new(0); -/// [`crate::g17::now_ns`] of the latest successful retirement, zero before the first. -static LAST_RETIRED_NS: AtomicU64 = AtomicU64::new(0); - -/// Called for every command that retires without error. -pub(crate) fn note_retired() { - RETIRED.fetch_add(1, Ordering::Relaxed); - LAST_RETIRED_NS.store(crate::g17::now_ns(), Ordering::Relaxed); -} - -pub(super) fn retired() -> u64 { - RETIRED.load(Ordering::Relaxed) -} - -/// Nanoseconds since any command last retired successfully, or `None` before the first. -pub(crate) fn since_retired(now: u64) -> Option { - match LAST_RETIRED_NS.load(Ordering::Relaxed) { - 0 => None, - last => Some(now.saturating_sub(last)), - } -} - -pub(super) struct Storm { - start: Option, - count: u8, - stopped: bool, - progress: u64, -} -impl Storm { - pub(super) const fn new() -> Self { - Self { start: None, count: 0, stopped: false, progress: 0 } - } - /// Like [`Self::note`], but work retired since the previous event starts a new window. - pub(super) fn note_unless_progress(&mut self, now: u64, progress: u64) -> bool { - if !self.stopped && progress != self.progress { - self.progress = progress; - self.start = None; - } - self.note(now) - } - pub(super) fn note(&mut self, now: u64) -> bool { - if self.stopped { return true; } - if self.start.is_none_or(|start| now.saturating_sub(start) >= WINDOW_NS) { - self.start = Some(now); - self.count = 0; - } - self.count += 1; - self.stopped = self.count >= LIMIT; - self.stopped - } -} - -#[kernel::macros::kunit_tests(asahi_g17_timeout_recovery_safety)] -mod tests { - use super::*; - use crate::g17::{preparation::Gate, recovery::State}; - use kernel::prelude::*; - - #[test] - fn consecutive_timeouts_during_recovery_are_bounded_and_lock_ordered() { - let gate = Gate::new().unwrap(); - let lease = gate.try_enter().unwrap(); - let state = KBox::pin_init(kernel::new_mutex!(State::new()), GFP_KERNEL).unwrap(); - gate.close(); - for n in 0..64 { - let stopped = { - let mut state = state.lock(); - state.request(0, -1).unwrap(); - assert!(state.pending()); - let stopped = state.timeout_limit(n * 2_000_000_000); - if stopped { gate.fail(); } // production state -> gate order - stopped - }; - assert_eq!(stopped, n >= u64::from(LIMIT - 1)); - } - // Recovery/stop must release state before joining the retained builder. - assert!(matches!(gate.wait_drained_until(crate::g17::now_ns() + 2_000_000), - Err(error) if error == ETIMEDOUT)); - assert!(!gate.reopen()); // failure never gives back an owned reservation - drop(lease); - gate.wait_drained_until(crate::g17::now_ns() + 100_000_000).unwrap(); - assert!(!gate.reopen()); - } - #[test] - fn successful_handshakes_do_not_reset_a_restart_storm() { - let mut state = State::new(); - for n in 0..64 { - assert_eq!(state.restart_limit(n), n >= u64::from(LIMIT - 1)); - } - assert!(state.restart_limit(u64::MAX)); - } - #[test] - fn separated_faults_do_not_accumulate_and_latched_loss_cannot_wrap() { - let mut storm = Storm::new(); - for n in 0..64 { assert!(!storm.note(n * WINDOW_NS)); } - for _ in 1..LIMIT { storm.note(64 * WINDOW_NS); } - assert!(storm.note(64 * WINDOW_NS)); - for _ in 0..1024 { assert!(storm.note(u64::MAX)); } - } - #[test] - fn recovery_table_ends_at_its_first_invalid_record() { - use crate::g17::recovery::Entries; - let mut entries = Entries::default(); - assert!(entries.include(1 | (12 << 4))); - assert!(entries.include(1 | 2 | (17 << 4))); - assert!(!entries.include(0)); - assert_eq!(entries.listed, 2); - assert_eq!(entries.guilty, 1u128 << 17); - assert_eq!(entries.cancelled, 0); - } - #[test] - fn replay_dependency_checks_never_wait_for_event_service() { - use crate::g17::dependency::replay_status; - assert_eq!(replay_status(0), Err(EBUSY)); - assert_eq!(replay_status(1), Ok(())); - assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED)); - assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED)); - assert_eq!(replay_status(ENODATA.to_errno()), Err(ECANCELED)); - } -} diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs index bd9d4c96be14f..307534708e922 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/slot.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -2,7 +2,7 @@ //! Bounded read-only slot identity sampling while firmware is halted. -use crate::mmu; +use crate::{hw::t8140, mmu}; use core::{ ffi::c_void, ptr::NonNull, @@ -19,11 +19,11 @@ impl Drop for Mapping { } } -pub(super) fn key(slot: usize, slot_keys: u64) -> Result { +pub(super) fn key(slot: usize) -> Result { if slot >= super::attribution::SLOTS { return Err(EINVAL); } - let physical = slot_keys + slot as u64 * size_of::() as u64; + let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; let offset = physical as usize & (mmu::UAT_PGSZ - 1); // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here // for mapping. Only the selected aligned eight-byte word is read below. @@ -31,7 +31,7 @@ pub(super) fn key(slot: usize, slot_keys: u64) -> Result { bindings::memremap( physical - offset as u64, mmu::UAT_PGSZ, - bindings::MEMREMAP_WB as usize, + bindings::MEMREMAP_WB as _, ) }; let mapping = Mapping(NonNull::new(raw).ok_or(ENOMEM)?); diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs deleted file mode 100644 index 7668871376bad..0000000000000 --- a/drivers/gpu/drm/asahi/g17/replay.rs +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Whether a pair's retained render groups, retired by firmware without executing, are -//! published again. Only the whole set is replayed, so no replayed group runs after a live -//! successor, and work a recovery blamed is never run again. - -/// One retained group, front first. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Group { - /// Published, not failed and not classified guilty by a recovery. - pub(crate) innocent: bool, - /// A recovery blamed it, whatever class it kept (an earlier recovery may have spared it): - /// it may have caused the fault. - pub(crate) blamed: bool, - /// Carries the strict abandonment witness. - pub(crate) abandoned: bool, - /// Already republished the maximum number of times. - pub(crate) exhausted: bool, - /// Retired by firmware after executing in full. - pub(crate) ran: bool, - /// Retired by firmware unexecuted, but never to run again: a recovery interrupted it after - /// its tiler started, so it may have run partly, or a command of another queue waited on it, - /// and that wait ended when firmware retired the group, before its output existed. - pub(crate) lost: bool, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Decision { - Replay, - /// A group may still execute: look again later. - Wait, - /// Never replay this set; the render timeout settles it. - Refuse, - /// Firmware has retired every group, but a dropped group cannot run again in order: a later - /// group already ran, or the dropped group had started or had a consumer. Fail the set now. - Fail, -} - -/// Groups are read in order and the first one that decides stops the walk, so a caller can -/// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its -/// successors, which consume its output, with it. An exhausted successor would break the order. -/// -/// After a restart the firmware retires the groups it had started on a pair without running -/// them, while groups it had not yet started run as usual. Once such a later group has run, a -/// replay of the dropped ones would run them after it; a group that had started cannot run -/// twice; and a group another queue waited on may already have been read before it ran. None of -/// these is replayed: the whole set fails at once instead of waiting for the render timeout. A -/// group that ran ahead of every dropped one is left to in-order retirement. -pub(crate) fn decide(groups: impl Iterator) -> Decision { - let mut front_exhausted = None; - let mut successor_exhausted = false; - let mut dropped = false; - let mut unordered = false; - for group in groups { - if !group.innocent || group.blamed { - return Decision::Refuse; - } - if group.ran { - if !dropped { - return Decision::Wait; - } - unordered = true; - continue; - } - if group.lost { - dropped = true; - unordered = true; - continue; - } - if !group.abandoned { - return Decision::Wait; - } - dropped = true; - match front_exhausted { - None => front_exhausted = Some(group.exhausted), - Some(_) => successor_exhausted |= group.exhausted, - } - } - if unordered { - return Decision::Fail; - } - match front_exhausted { - Some(true) => Decision::Replay, - Some(false) if !successor_exhausted => Decision::Replay, - _ => Decision::Refuse, - } -} - -#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_render_replay))] -mod tests { - use super::*; - fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { - Group { innocent, blamed: false, abandoned, exhausted, ran: false, lost: false } - } - const OK: Group = - Group { innocent: true, blamed: false, abandoned: true, exhausted: false, ran: false, lost: false }; - const BLAMED: Group = Group { blamed: true, ..OK }; - const RAN: Group = Group { abandoned: false, ran: true, ..OK }; - const LOST: Group = Group { abandoned: false, lost: true, ..OK }; - #[test] - fn every_group_abandoned_and_innocent_is_replayed() { - assert_eq!(decide([OK].into_iter()), Decision::Replay); - assert_eq!(decide([OK, OK, OK].into_iter()), Decision::Replay); - } - #[test] - fn nothing_retained_is_never_replayed() { - assert_eq!(decide([].into_iter()), Decision::Refuse); - } - #[test] - fn guilty_work_is_never_replayed() { - assert_eq!(decide([group(false, true, false)].into_iter()), Decision::Refuse); - assert_eq!(decide([OK, group(false, true, false)].into_iter()), Decision::Refuse); - } - #[test] - fn blamed_work_is_never_replayed_even_when_spared() { - // An earlier recovery spared it, so it still counts as innocent; the blame refuses it. - assert_eq!(decide([BLAMED].into_iter()), Decision::Refuse); - assert_eq!(decide([BLAMED, OK].into_iter()), Decision::Refuse); - assert_eq!(decide([OK, BLAMED].into_iter()), Decision::Refuse); - // Also once the replay limit is reached, where an innocent front alone is replayed. - let exhausted = Group { exhausted: true, ..BLAMED }; - assert_eq!(decide([exhausted].into_iter()), Decision::Refuse); - } - #[test] - fn a_live_group_waits_and_stops_the_walk() { - assert_eq!(decide([OK, group(true, false, false)].into_iter()), Decision::Wait); - // Nothing after the live group is looked at, guilty or not. - assert_eq!( - decide([group(true, false, false), group(false, true, false)].into_iter()), - Decision::Wait - ); - let mut seen = 0; - let groups = [group(true, false, false), OK, OK].into_iter().inspect(|_| seen += 1); - assert_eq!(decide(groups), Decision::Wait); - assert_eq!(seen, 1); - } - #[test] - fn an_exhausted_successor_refuses_an_exhausted_front_replays() { - assert_eq!(decide([OK, group(true, true, true)].into_iter()), Decision::Refuse); - assert_eq!(decide([group(true, true, true), OK].into_iter()), Decision::Replay); - assert_eq!( - decide([group(true, true, true), group(true, true, true)].into_iter()), - Decision::Replay - ); - } - #[test] - fn a_group_that_ran_after_dropped_ones_fails_the_set() { - // Seen after restarts: the started groups were dropped, the queued ones ran. - assert_eq!(decide([OK, OK, RAN, RAN].into_iter()), Decision::Fail); - assert_eq!(decide([OK, RAN, OK].into_iter()), Decision::Fail); - // Also with an exhausted front, which a replay would only fail later. - assert_eq!(decide([group(true, true, true), RAN].into_iter()), Decision::Fail); - } - #[test] - fn a_group_that_ran_first_is_left_to_retirement() { - assert_eq!(decide([RAN].into_iter()), Decision::Wait); - assert_eq!(decide([RAN, OK].into_iter()), Decision::Wait); - } - #[test] - fn a_started_dropped_group_fails_the_set() { - assert_eq!(decide([OK, LOST].into_iter()), Decision::Fail); - assert_eq!(decide([LOST, OK].into_iter()), Decision::Fail); - } - #[test] - fn failing_still_waits_for_live_groups_and_refuses_guilty_ones() { - assert_eq!(decide([OK, RAN, group(true, false, false)].into_iter()), Decision::Wait); - assert_eq!(decide([OK, RAN, group(false, true, false)].into_iter()), Decision::Refuse); - assert_eq!(decide([LOST, BLAMED].into_iter()), Decision::Refuse); - } -} diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d391fbb3001bb..ffaff6d80dbf9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -25,38 +25,14 @@ use super::{ kick, qos, queue::{self, admission, compute}, }; -use super::preparation::Admission; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; +const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; -/// Pending admission keeps exactly retired compute backing available for reuse. -/// The guard spans retries and disappears on publication, cancellation or error. -/// It borrows Shared, so the accounting cannot retain the device after teardown. -struct ComputeDemand<'a>(&'a Arc); - -impl<'a> ComputeDemand<'a> { - fn new(shared: &'a Arc) -> Self { - // Each admitted logical queue has at most one scheduler publisher. - // The execution-context bound is far below the counter's range. - shared.compute_demand.fetch_add(1, Ordering::AcqRel); - Self(shared) - } -} - -impl Drop for ComputeDemand<'_> { - fn drop(&mut self) { - if self.0.compute_demand.fetch_sub(1, Ordering::AcqRel) == 1 { - // Reinstate normal idle retention even if the last publisher was - // cancelled before a kick could produce another event. - self.0.queue_events(); - } - } -} - /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. #[derive(Clone, Copy)] @@ -68,103 +44,13 @@ pub(super) struct ComputeReservation { struct ComputeEntry { reservation: ComputeReservation, - queue: ComputeSlot, -} - -/// A physical compute queue behind its own lock. Publication into a bound queue takes only -/// this lock; device-mutex holders take it to observe or change the queue (device mutex -> -/// cell). A logical queue may keep the cell after the registry gave the queue up: the cell is -/// then empty, so the cached reference never extends any firmware-visible lifetime. -#[pin_data] -pub(super) struct ComputeCell { - #[pin] - queue: kernel::sync::Mutex>>, -} - -impl ComputeCell { - fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { - queue <- kernel::new_mutex!(None, "g17::ComputeCell::queue"), - }), - GFP_KERNEL, - ) - } - - /// The installed queue, locked; None once the registry gave it up. - pub(super) fn lock(&self) -> Option> { - QueueGuard::new(self.queue.lock()) - } -} - -/// A locked installed queue. -pub(super) struct QueueGuard<'a> { - queue: core::ptr::NonNull, - _guard: kernel::sync::MutexGuard<'a, Option>>, -} - -impl<'a> QueueGuard<'a> { - fn new(mut guard: kernel::sync::MutexGuard<'a, Option>>) -> Option { - let queue = core::ptr::NonNull::from((*guard).as_deref_mut()?); - Some(Self { queue, _guard: guard }) - } -} - -impl core::ops::Deref for QueueGuard<'_> { - type Target = compute::Queue; - fn deref(&self) -> &compute::Queue { - // SAFETY: The boxed queue cannot be taken or dropped while the guard holds its lock. - unsafe { self.queue.as_ref() } - } -} - -impl core::ops::DerefMut for QueueGuard<'_> { - fn deref_mut(&mut self) -> &mut compute::Queue { - // SAFETY: As for `deref`; the lock makes this the only reference. - unsafe { self.queue.as_mut() } - } -} - -/// The registry's handle on one compute cell. Dropping it drops the queue in the cell, so a -/// cached cell never keeps a removed queue (or its QID's aliases) alive. -struct ComputeSlot(Arc); - -impl ComputeSlot { - fn new() -> Result { - Ok(Self(ComputeCell::new()?)) - } - fn as_deref(&self) -> Option> { - self.0.lock() - } - fn as_deref_mut(&mut self) -> Option> { - self.0.lock() - } - fn is_some(&self) -> bool { - self.0.queue.lock().is_some() - } - fn is_none(&self) -> bool { - !self.is_some() - } - /// Gives the queue up to the caller, which drops it outside the cell lock. - fn take(&mut self) -> Option> { - self.0.queue.lock().take() - } - fn put(&mut self, queue: Option>) { - let old = core::mem::replace(&mut *self.0.queue.lock(), queue); - drop(old); - } -} - -impl Drop for ComputeSlot { - fn drop(&mut self) { - let queue = self.0.queue.lock().take(); - drop(queue); - } + queue: Option>, } pub(super) struct Registry { pub(super) qids: kick::QueueIds, pub(super) buffers: BufferIds, + pub(super) accounting: qos::Accounting, pub(super) metrics: Arc, pub(super) qos_ids: Arc, pub(super) admission: Arc, @@ -175,10 +61,6 @@ pub(super) struct Registry { compute_scheduler_building: bool, replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, - /// Completion-ring slot where the next compute scan starts. - completion_cursor: usize, - /// Control word of each completion-ring slot as of the last scan that read it. - completion_seen: KVVec, worker_deferred: Option, teardown: super::teardown::Pending, } @@ -186,35 +68,22 @@ pub(super) struct Registry { impl Registry { /// Initialize inside the firmware allocation instead of staging the whole /// registry on the probe stack. No additional allocation is required. - #[cfg(CONFIG_KUNIT = "y")] pub(super) fn new( clusters: u32, - deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], - variant: &'static crate::g17::fw::Variant, - ) -> impl Init { - Self::with_waiters(clusters, deflake, descriptor_flags, variant, None, None) - } - - /// `queues` wakes logical-queue creation when a logical queue slot is returned; - /// `renders` wakes render publication when a render pair or lease returns. - pub(super) fn with_waiters( - clusters: u32, - deflake: crate::g17::fw::render::Deflake, - descriptor_flags: [u32; 2], - variant: &'static crate::g17::fw::Variant, - queues: Option>, - renders: Option>, ) -> impl Init { kernel::try_init!(Self { - qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { - qids.reserve_initial_render() + qids <- kick::QueueIds::new().chain(|qids| { + qids.reserve(1, DataMaster::Tiling, Some(0))?; + qids.reserve(2, DataMaster::Fragment, Some(1))?; + Ok(()) }), buffers <- BufferIds::new(), + accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, - render: render::RenderState::new(clusters, deflake, descriptor_flags)?, + admission: admission::Pool::new()?, + render: render::RenderState::new(clusters, descriptor_flags)?, identities: queue::Identities::new(), compute: { let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; @@ -227,21 +96,12 @@ impl Registry { compute_scheduler_building: false, replay_outer_wait: None, observations: compute::retirement::Batch::new()?, - completion_cursor: 0, - completion_seen: { - let slots = super::initdata::COMPUTE_COMPLETION_SLOTS; - let mut seen = KVVec::with_capacity(slots, GFP_KERNEL)?; - for _ in 0..slots { - seen.push_within_capacity(0).map_err(|_| ENOMEM)?; - } - seen - }, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) } - pub(super) fn compute(&self, owner: u64) -> Option> { + pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { self.compute .iter() .flatten() @@ -249,24 +109,19 @@ impl Registry { .find(|queue| queue.owner() == Some(owner)) } - pub(super) fn compute_mut(&mut self, owner: u64) -> Option> { - self.compute(owner) - } - - /// The cell of `owner`'s bound queue, which its publisher may lock without the device mutex. - pub(super) fn compute_cell(&self, owner: u64) -> Option> { - self.compute.iter().flatten().find_map(|entry| { - entry.queue.as_deref()?.owner().filter(|bound| *bound == owner)?; - Some(entry.queue.0.clone()) - }) + pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { + self.compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) } /// An installed idle queue keeps its original QID and backing. Mapping a /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result { - admission.check()?; + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize) -> Result { if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { if queue.pool_fits(required) { Rebind::Done } else { Rebind::Grow } @@ -274,102 +129,23 @@ impl Registry { Rebind::Fresh }); } - // A binding persists for its logical queue's lifetime while physical - // QIDs allow: take a channel of a closed owner, then a fresh QID, and - // only then the least recently idle channel of one open owner. - if let Some(rebind) = self.bind_reusable(owner, context, required, admission)? { - return Ok(rebind); - } - if self.qids.compute_fits(self.admission.queues() as usize) { - return Ok(Rebind::Fresh); - } - let teardown = &self.teardown; - let idle = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| queue.shareable()) - .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) - .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) - .min_by_key(|(key, _)| *key) - .map(|(_, qid)| qid); - let Some(qid) = idle else { return Err(ENOSPC); }; - admission.check()?; - let shared = self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.qid() == qid) - .ok_or(EIO)? - .share_idle()?; - if !shared { - return Err(ENOSPC); - } - Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) - } - - /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks - /// only whether some channel is free for a new owner. - fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { - if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { - return if queue.spared_pending() { - Readiness::Progress - } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { - // Publication reports the mismatch. - Readiness::Ready - } else if queue.pool_fits(required) || !queue.growing() { - Readiness::Ready - } else { - Readiness::Progress - }; - } - let teardown = &self.teardown; - let available = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| { - (queue.reusable() || queue.shareable() || queue.can_rearm()) - && !queue.context().is_some_and(|bound| teardown.releasing(bound)) - }); - if available || self.qids.compute_fits(self.admission.queues() as usize) { - Readiness::Ready - } else { - Readiness::Channel - } - } - - /// Binds `owner` to a reusable channel whose last owner released it, or - /// names a vacant one to re-arm. None if no channel is reusable. - fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { let mut vacant = None; - let teardown = &self.teardown; - for mut queue in self + for queue in self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .filter(|queue| queue.reusable()) { - admission.check()?; - // The release of the scheduler these records were bound to is armed - // or not yet consumed: the firmware may process it against this - // queue, which therefore must not change owner until it has. - if queue.context().is_some_and(|bound| teardown.releasing(bound)) { - continue; - } if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; } if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; - return Ok(Some(Rebind::Done)); + return Ok(Rebind::Done); } - Ok(vacant.map(Rebind::Rearm)) + Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -380,7 +156,7 @@ impl Registry { pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { - let mut queue = self + let queue = self .compute .iter_mut() .flatten() @@ -391,18 +167,13 @@ impl Registry { } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { - let teardown = &self.teardown; - let mut queue = self + let queue = self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid && queue.reusable()) .ok_or(EAGAIN)?; - // Its last owner may have closed while the backing was rebuilt. - if queue.context().is_some_and(|bound| teardown.releasing(bound)) { - return Err(EAGAIN); - } queue.bind_owner(owner, context.clone()) } @@ -460,7 +231,7 @@ impl Registry { let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { - let mut queue = self + let queue = self .compute .iter_mut() .flatten() @@ -490,13 +261,12 @@ impl Registry { let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); let mut freed = 0; - for mut queue in self + for queue in self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - if !deferred.vacated_room() { break; } if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { deferred.defer_vacated(vacated); freed += 1; @@ -505,23 +275,12 @@ impl Registry { Ok(freed) } - /// Builders and released queues waiting for their exact retirement or pool - /// witness temporarily occupy identities. A failed admission must let those - /// transactions finish rather than fail a healthy successor immediately. - fn compute_admission_error(&self, error: Error) -> Error { - if error == ENOSPC { - EAGAIN - } else { - error - } - } - pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { if owner == 0 || self.compute.iter().flatten().any(|entry| { entry .queue - .as_deref() + .as_ref() .map_or(entry.reservation.owner == owner, |queue| { queue.owner() == Some(owner) }) @@ -533,19 +292,15 @@ impl Registry { .compute .iter() .position(Option::is_none) - .ok_or_else(|| self.compute_admission_error( - crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; + .ok_or(ENOSPC)?; let identity = self.identities.reserve(1)?.start; - let queues = self.admission.queues() as usize; let id = self .qids - .reserve_for(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID), queues) - .map_err(|error| self.compute_admission_error(error))?; - self.note_build_room(); + .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, - queue: ComputeSlot::new()?, + queue: None, }); Ok(reservation) } @@ -577,7 +332,7 @@ impl Registry { return Err(EIO); } let entry = self.reserved(reservation)?; - entry.queue.put(queue.take()); + entry.queue = queue.take(); Ok(()) } @@ -585,7 +340,6 @@ impl Registry { self.reserved(reservation)?; self.qids.cancel_unpublished(reservation.id)?; self.compute[reservation.slot] = None; - self.note_build_room(); Ok(()) } @@ -593,32 +347,20 @@ impl Registry { let Some(slot) = self.compute.iter().position(|entry| { entry .as_ref() - .and_then(|entry| entry.queue.as_deref()) + .and_then(|entry| entry.queue.as_ref()) .is_some_and(|queue| queue.owner() == Some(owner)) }) else { return Ok(()); }; let entry = self.compute[slot].as_mut().ok_or(EIO)?; - let unpublished = entry.queue.as_deref_mut().ok_or(EIO)?.release_owner()?; - if unpublished { + if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { self.qids.cancel_unpublished(entry.reservation.id)?; self.compute[slot] = None; - self.note_build_room(); } Ok(()) } } -/// What a compute packet still lacks before publication. -#[derive(Copy, Clone, PartialEq, Eq)] -pub(super) enum Readiness { - Ready, - /// A physical channel: every one is busy and no QID fits. - Channel, - /// Retirement-bound capacity or reopened publication. - Progress, -} - /// Outcome of offering an owner a retained compute queue. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Rebind { @@ -631,56 +373,13 @@ pub(super) enum Rebind { Fresh, } - -#[kunit_tests(asahi_g17_uninstalled_cleanup)] -mod uninstalled_cleanup_tests { - use super::*; - use crate::g17::fw::render::Deflake; - use crate::g17::fw::variant::V26A428; - #[test] - fn empty_reservation_prevents_reuse_until_explicit_cancellation() { - let mut registry = KBox::init( - Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), - GFP_KERNEL, - ) - .unwrap(); - let first = registry.reserve_compute(1).unwrap(); - assert!(registry.reserved(first).is_ok()); - assert!(registry.reserve_compute(1).is_err()); - let second = registry.reserve_compute(2).unwrap(); - assert_ne!(first.id.qid(), second.id.qid()); - registry.cancel_compute(first).unwrap(); - assert!(registry.reserved(first).is_err()); - let next = registry.reserve_compute(3).unwrap(); - assert_eq!(first.id.qid(), next.id.qid()); - assert!(first.id != next.id); - } -} - /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. -#[pin_data] pub(super) struct Backend { shared: Arc, pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, - compute_backlog: core::sync::atomic::AtomicU32, - closed: core::sync::atomic::AtomicBool, - /// The cell of the compute queue this owner last published to under the device mutex, - /// with the shared compute objects mapped into its VM. Direct publication revalidates the - /// binding under the cell lock; a stale cell only sends the packet back to that path. - #[pin] - direct: kernel::sync::Mutex>>, - /// The channel a prepared packet is about to publish into. Its claim keeps the idle - /// channel from being shared away between prepare_job and run_job. - #[pin] - claimed: kernel::sync::Mutex>>, - /// Render pair admission, shared by every logical queue of the session. - admission: Arc, - /// The render pair this owner last published to without the device mutex - /// (zero for none): a hint, revalidated by every lend. - direct_slot: core::sync::atomic::AtomicU32, } impl Backend { @@ -690,63 +389,24 @@ impl Backend { owner: u64, policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { - // A closed queue keeps its application context until its scheduler - // release is consumed and the context is returned. When only such - // contexts block this one, wait (bounded) for the teardown instead of - // failing a live client; the preparation lease is not held while - // sleeping, and nothing of the new queue is visible yet. - const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; - let started = now_ns(); - let (lease, uat, pool, qos, execution, slot) = loop { - let observed = shared.queues.observe(); - let lease = shared.preparations.enter(true)?; + let lease = shared.preparations.enter(true)?; + let dev = shared.drm()?; + let (uat, pool, qos, execution, slot) = { let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; - // Installed compute channels narrow render's QIDs: every open - // logical queue keeps a dedicated render pair within reach. - if !firmware.queues.qids.render_fits(firmware.queues.admission.queues() as usize) { - return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); - } - match firmware.uat.new_execution_context(vm) { - Ok(execution) => { - break ( - lease, - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) - } - Err(error) if error == ENOSPC => { - // Every open queue holds one context; this one is counted - // in `queues()` already but holds none yet. - let open = firmware.queues.admission.queues().saturating_sub(1); - let closing = firmware.uat.execution_contexts_used().saturating_sub(open); - if closing == 0 || now_ns().saturating_sub(started) >= RETAINED_CONTEXT_WAIT_NS { - return Err(super::capacity::refused(super::capacity::Site::ExecutionContexts)); - } - drop(slot); - drop(state); - drop(lease); - // Nothing is visible yet: a signalled creator restarts. - if kernel::current!().signal_pending() { - return Err(ERESTARTSYS); - } - shared.queue_events(); - // A closing queue returns its context when its release - // is reclaimed; that reclaim wakes this creator. - let left = RETAINED_CONTEXT_WAIT_NS.saturating_sub(now_ns().saturating_sub(started)); - shared.queues.block(observed, left.div_ceil(1_000_000) as u32, true)?; - } - Err(error) => return Err(error), - } + let execution = firmware.uat.new_execution_context(vm)?; + ( + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) }; - let dev = shared.drm()?; let alloc = super::object::Allocator { dev: &dev, uat: &uat, @@ -754,16 +414,7 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new( - &alloc, - &pool, - &qos, - execution, - owner_pid, - policy, - shared.renders.clone(), - shared.progress.clone(), - )?, + Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -771,21 +422,14 @@ impl Backend { return Err(EAGAIN); } drop(state); - let admission = slot.pool().clone(); Ok(( - Arc::pin_init( - pin_init!(Self { + Arc::new( + Self { shared: shared.clone(), context, owner, backlog: core::sync::atomic::AtomicU32::new(0), - compute_backlog: core::sync::atomic::AtomicU32::new(0), - closed: core::sync::atomic::AtomicBool::new(false), - direct <- kernel::new_mutex!(None, "g17::Backend::direct"), - claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), - admission, - direct_slot: core::sync::atomic::AtomicU32::new(0), - }), + }, GFP_KERNEL, )?, slot, @@ -794,15 +438,13 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. - fn shared_compute(&self, admission: &Admission<'_>) -> Result { + fn shared_compute(&self) -> Result { loop { - admission.check()?; - let lease = self.shared.preparations.enter_until(admission)?; + let lease = self.shared.preparations.enter(false)?; let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); loop { - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -821,9 +463,7 @@ impl Backend { firmware.queues.compute_building = true; break firmware.uat.clone(); } - let _ = self.shared.changed.wait_timeout( - &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) - ); + self.shared.changed.wait(&mut state); } }; let alloc = super::object::Allocator { @@ -834,9 +474,7 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if let Err(error) = admission.check() { - Err(error) - } else if !lease.is_current() { + if !lease.is_current() { Err(EAGAIN) } else { built @@ -855,7 +493,7 @@ impl Backend { .queues .compute_building = false; } - self.shared.note_changed(); + self.shared.changed.notify_all(); drop(lease); if result != Err(EAGAIN) { return result; @@ -869,12 +507,10 @@ impl Backend { &self, alloc: &super::object::Allocator<'_>, lease: &super::preparation::Lease, - admission: &Admission<'_>, ) -> Result { { let mut state = self.shared.state.lock(); loop { - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); @@ -886,18 +522,14 @@ impl Backend { firmware.queues.compute_scheduler_building = true; break; } - let _ = self.shared.changed.wait_timeout( - &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) - ); + self.shared.changed.wait(&mut state); } } let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if let Err(error) = admission.check() { - Err(error) - } else if !lease.is_current() { + if !lease.is_current() { Err(EAGAIN) } else { built @@ -915,30 +547,12 @@ impl Backend { .queues .compute_scheduler_building = false; } - self.shared.note_changed(); + self.shared.changed.notify_all(); result } - /// The caller keeps its preparation lease through this two-phase rollback. - /// An empty registry entry prevents reusing the QID/free-list identity while - /// its old aliases still occupy canonical or client addresses. Recovery joins - /// this lease before removing the session, even if its epoch was invalidated. - fn finish_uninstalled_compute( - &self, retired: Option<(ComputeReservation, KBox)>, - ) -> Result { - let Some((reservation, queue)) = retired else { return Ok(()); }; - drop(queue); - let mut state = self.shared.state.lock(); - let firmware = state.as_deref_mut().ok_or(ENODEV)?; - firmware.queues.cancel_compute(reservation)?; - drop(state); - // The slot and its never-published QID are free again. - self.shared.channels.notify(); - Ok(()) - } - - fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { - admission.check()?; + fn grow_compute(&self, required: usize) -> Result { + use kernel::time::{delay::fsleep, Delta}; { let state = self.shared.state.lock(); let firmware = state.as_ref().ok_or(ENODEV)?; @@ -946,83 +560,45 @@ impl Backend { return Ok(()); } } - let lease = self.shared.preparations.enter_until(admission)?; + let lease = self.shared.preparations.enter(false)?; let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; - let mut growth = None; - let result = (|| loop { - // Growth waits for this queue's retirement and for firmware to consume the pool - // release; both are progress the event worker reports. - let observed = self.shared.progress.observe(); + let started = now_ns(); + loop { let result = (|| { let mut state = self.shared.state.lock(); - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let mut queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; + let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; if queue.pool_fits(required) { return Ok(None); } - if growth.is_none() { - let notify = queue.begin_growth(&firmware.init, required)?; - // Capture physical identity before any following fallible - // operation. Close can remove the owner-keyed lookup. - growth = queue.growth_token(); - if notify { - fence(Ordering::SeqCst); - firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; - } + if queue.begin_growth(&firmware.init, required)? { + fence(Ordering::SeqCst); + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; } queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) })(); deferred.finish(); match result { Ok(old) => { drop(old); return Ok(()); } - Err(error) if error == EAGAIN && lease.is_current() => { - self.shared.queue_events(); - self.wait_release(&self.shared.progress, observed, admission)?; - } - Err(error) => return Err(error), - } - })(); - if result.is_err() { - if let Some(token) = growth { - let mut retired = None; - let mut state = self.shared.state.lock(); - if let Some(firmware) = state.as_deref_mut() { - for index in 0..firmware.queues.compute.len() { - let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; - let Some(aborted) = entry.queue.as_deref_mut() - .map(|mut queue| queue.abort_growth(token)).transpose()? else { continue; }; - if aborted == Some(true) { - // Keep the empty entry and QID reserved until all - // graph aliases have been destroyed outside the lock. - retired = entry.queue.take().map(|queue| (entry.reservation, queue)); - break; - } + Err(error) if error == EAGAIN && lease.is_current() + && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); } - } - drop(state); - // Tentative replacement mappings also belong to this builder. - // Discard them before a fresh QID can acquire a successor. - drop(alias.take()); - drop(pages.take()); - self.finish_uninstalled_compute(retired)?; - self.shared.queue_events(); + Err(error) => return Err(error), } } - result } - fn build_compute(&self, required: usize, admission: &Admission<'_>) -> Result { - admission.check()?; - self.admission_open()?; + fn build_compute(&self, required: usize) -> Result { loop { - match self.shared_compute(admission) { + match self.shared_compute() { Err(error) if error == EAGAIN => continue, result => { result?; @@ -1031,14 +607,10 @@ impl Backend { } } loop { - admission.check()?; - // Sampled before any capacity check of this attempt. - let observed = self.shared.channels.observe(); - let lease = self.shared.preparations.enter_until(admission)?; + let lease = self.shared.preparations.enter(false)?; let dev = self.shared.drm()?; - let (uat, objects, reservation) = { + let (uat, reservation) = { let mut state = self.shared.state.lock(); - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -1047,44 +619,13 @@ impl Backend { { return Err(EIO); } - self.admission_open()?; - let rebound = match firmware.queues.rebind_compute(self.owner, &self.context, required, admission) { - Err(error) if error == ENOSPC => { - drop(state); - drop(lease); - self.wait_channels(observed, admission, true)?; - continue; - } - result => result?, - }; - match rebound { + match firmware.queues.rebind_compute(self.owner, &self.context, required)? { Rebind::Done => return Ok(()), - Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } - Rebind::Rearm(qid) => (firmware.uat.clone(), firmware.pool.clone(), Err(qid)), + Rebind::Grow => { drop(state); return self.grow_compute(required); } + Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { - let reservation = match firmware.queues.reserve_compute(self.owner) { - Ok(reservation) => reservation, - Err(error) if error == EAGAIN => { - // All physical channels are occupied. Retain their - // USC backing for exact-retirement handoff; releasing - // it cannot create a QID and forces fresh allocation. - drop(state); - drop(lease); - self.wait_channels(observed, admission, false)?; - continue; - } - Err(error) if error == ENOSPC => { - let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; - drop(state); - if reclaimed.is_none() { - return Err(error); - } - drop(reclaimed); - continue; - } - Err(error) => return Err(error), - }; - (firmware.uat.clone(), firmware.pool.clone(), Ok(reservation)) + let reservation = firmware.queues.reserve_compute(self.owner)?; + (firmware.uat.clone(), Ok(reservation)) } } }; @@ -1097,32 +638,13 @@ impl Backend { Err(qid) => { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. - let built = (|| -> Result<_> { - let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) - })(); - let (pages, ring) = match built { - Err(error) if error == ENOSPC => { - drop(lease); - self.wait_channels(observed, admission, true)?; - continue; - } - result => result?, - }; + let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; + let ring = compute::DescriptorRing::new(&alloc)?; let mut state = self.shared.state.lock(); - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { continue; } - if let Err(error) = self.admission_open() { drop(state); return Err(error); } - // A competing ready owner may have taken/rearmed it while - // backing was built off-lock. Leave rejected backing local - // so it is dropped after the device guard, not in rearm(). - let ready = firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.qid() == qid && queue.can_rearm()); - if !ready { drop(state); continue; } let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, @@ -1138,12 +660,10 @@ impl Backend { let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; { let mut state = self.shared.state.lock(); - admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } - self.admission_open()?; firmware.queues.reserved(reservation)?; let (slot, page_list) = pool.pool_descriptor()?; firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; @@ -1151,16 +671,8 @@ impl Backend { let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease, admission)?; - let queue = compute::Queue::new( - &alloc, - &objects, - backing, - self.owner, - self.context.clone(), - self.shared.variant.pointers, - self.shared.operand_state, - )?; + self.compute_scheduler(&alloc, &lease)?; + let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); let (mut built, build_error) = match built { @@ -1170,14 +682,12 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if let Err(error) = admission.check() { - Err(error) - } else if !lease.is_current() { + if !lease.is_current() { Err(EAGAIN) } else if let Some(error) = build_error { Err(error) } else { - admission.check().and_then(|()| firmware.queues.install_compute(reservation, &mut built)) + firmware.queues.install_compute(reservation, &mut built) } }; // The old aliases must disappear before a waiting constructor can @@ -1193,11 +703,9 @@ impl Backend { .cancel_compute(reservation)?; } drop(lease); - if result == Err(ENOSPC) { - self.wait_channels(observed, admission, true)?; - continue; + if result != Err(EAGAIN) { + return result; } - if result != Err(EAGAIN) { return result; } } } } @@ -1206,7 +714,6 @@ impl Backend { /// The physical namespace bounds the batch independently of open logical queues. /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { - has_source_completions: bool, values: KVVec, /// Exited owners detached from idle retained compute queues and the backing /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. @@ -1242,7 +749,6 @@ impl DeferredBatch { 0 }; Ok(Self { - has_source_completions: false, values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, @@ -1274,9 +780,6 @@ impl DeferredBatch { drop(release); } } - pub(super) fn vacated_room(&self) -> bool { - self.vacated.len() < self.vacated.capacity() - } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -1296,18 +799,10 @@ impl DeferredBatch { // it. Expose terminal VM status now, before off-lock fence callbacks. // Recovery-spared and cancelled commands never poison their VM. if let Deferred::FailedOwned(completion, error) = &value { - if completion.poisons(*error) { + if *error != ECANCELED && !completion.spared() { completion.status().record(*error); } } - match &value { - Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) - | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => - { completion.note_input_error(*error); }, - Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => - completion.note_source_retired(), - _ => {}, - } self.values.push_within_capacity(value).map_err(|_| EIO)?; // A later publisher can borrow this pair before finish signals its old // fence. Publish the exact completion's witness only after collection @@ -1319,33 +814,11 @@ impl DeferredBatch { ) = self.values.last() { completion.note_retirement_queued(); - self.has_source_completions |= completion.sources().is_some(); } Ok(()) } - /// Completions collected so far. - pub(super) fn pending(&self) -> usize { - self.values.len() - } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { - use super::completion::Deferred; - if self.has_source_completions { - for _ in 0..self.values.len() { - let mut changed = false; - for value in &self.values { - if let Deferred::Retired(completion, Ok(_)) - | Deferred::Drained(completion, Ok(_)) = value - { - if let Err(error) = completion.check_source_inputs() { - changed |= completion.note_input_error(error); - } - } - } - if !changed { break; } - } - } - self.has_source_completions = false; for value in self.values.drain_all() { value.finish(); } @@ -1366,15 +839,15 @@ impl DeferredBatch { struct ComputeHost<'a> { init: &'a super::initdata::InitData, - port: &'a super::port::Port, - outer: Option>, + primary: &'a mut super::Coprocessor, effort: &'a mut super::power::Effort, qids: &'a mut kick::QueueIds, + accounting: &'a mut qos::Accounting, generation: u64, } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - Ok(self.port.next_publication()) + self.qids.next_compute_publication() } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -1391,204 +864,82 @@ impl compute::Host for ComputeHost<'_> { scheduler: u64, policy: super::fw::queue::Policy, ) -> Result { - self.port.qos_publish(owner, scheduler, policy) + self.accounting.publish( + &qos::View::new(self.init.qos()?)?, + owner, + scheduler, + policy, + qos::clock, + ) } fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.port.qos_cancel(publication) + self.accounting + .cancel(&qos::View::new(self.init.qos()?)?, publication) } fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) + fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.init.publish_work(priority, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { - self.port.notify(message) + self.primary.notify(message) } fn note_submission(&mut self) -> Result { self.effort.submit(); - super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) - } -} - -/// Publication into a bound, installed queue through the port alone. The caller checked the -/// recovery state, full effort and the queue's binding; first installation never comes here. -struct DirectHost<'a> { - port: &'a super::port::Port, - outer: Option>, - generation: u64, -} -impl compute::Host for DirectHost<'_> { - fn next_compute_publication(&mut self) -> Result { - Ok(self.port.next_publication()) - } - fn epoch(&self) -> Result<(u64, u32)> { - Ok((self.generation, self.port.recovery_state()?)) - } - fn prepare_compute_shared(&mut self) -> Result { - Err(EAGAIN) - } - fn qos_publish( - &mut self, - owner: qos::Owner, - scheduler: u64, - policy: super::fw::queue::Policy, - ) -> Result { - self.port.qos_publish(owner, scheduler, policy) - } - fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.port.qos_cancel(publication) - } - fn publish_qid(&mut self, _id: kick::Id) -> Result { - // An installed queue published its identity with its first command. - Ok(()) - } - fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) - } - fn notify(&mut self, message: u64) -> Result { - self.port.notify(message) - } - fn note_submission(&mut self) -> Result { - Ok(()) + super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) } } impl super::Firmware { - #[inline] - fn publish_compute(&mut self, owner: u64, packet: Arc, - dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result - { - if packet.completion.sources().is_some() { - self.publish_compute_inputs(owner, packet, dependencies, deferred) - } else { - self.publish_compute_raw(owner, packet, dependencies, &[], deferred) - } - } - - #[inline(never)] - fn publish_compute_inputs(&mut self, owner: u64, packet: Arc, - dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result - { - if self.recovery.pending() || super::recovery::Memory::recovery_state(&self.init)? != 0 { - return Err(EAGAIN); - } - let sources = packet.completion.sources().ok_or(EINVAL)?.clone(); - sources.check()?; - let mut merged = super::dependency::Frontier::new(); - for dependency in dependencies { merged.insert(0, *dependency)?; } - let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; - let mut native: KVec = - KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; - for source in &sources.sources { - if source.submission.source_status()? { continue; } - let found = self.queues.compute.iter_mut().flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find_map(|mut queue| match queue.source_dependency(source, packet.completion.status(), - self.recovery.generation()) { - Ok(None) => None, - result => Some(result), - }).ok_or(EAGAIN)??; - let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { - return Err(EAGAIN); - }; - // The completion-wait record orders the consumer behind its - // source; naming the source in the kick entry as well only adds - // a second, slower wait and a predecessor record. - let wait = super::sync_input::NativeWait { sequence, dependency, stamp_va, event_slot }; - if let Some(old) = native.iter_mut().find(|old| - old.dependency.qid() == dependency.qid()) { - if sequence > old.sequence { *old = wait; } - } else { - native.push(wait, GFP_KERNEL)?; - } - holds.push(hold, GFP_KERNEL)?; - } - let result = self.publish_compute_raw(owner, packet.clone(), merged.as_slice(), - native.as_slice(), deferred); - if packet.is_published() { - for queue in self.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) - { - for source in &sources.sources { queue.note_input_consumer(source); } - } - sources.set_holds(holds); - super::sync_input::note_published(); - super::sync_input::note_native(native.len()); - } - result - } - - fn publish_compute_raw( + fn publish_compute( &mut self, owner: u64, packet: Arc, dependencies: &[super::fw::kick::KickDependency], - native: &[super::sync_input::NativeWait], deferred: &mut DeferredBatch, ) -> Result { if self.recovery.pending() { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // reservation; dropping it unused returns the work-ring slot. - let outer = self.port.reserve_compute(packet.context.policy().priority())?; + // capacity check. The same device lock covers queue.publish below. + if !self + .init + .work_ready(packet.context.policy().priority(), DataMaster::Compute)? + { + return Err(EAGAIN); + } let queues = &mut self.queues; - let mut queue = queues + let queue = queues .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.owner() == Some(owner)) .ok_or(EIO)?; - // The first command after an owner change waits while the scheduler - // the queue was bound to is being released: a live previous owner may - // have closed after giving up the queue. - if queue - .pending_switch_from() - .is_some_and(|previous| queues.teardown.releasing(previous)) - { - return Err(EAGAIN); - } let mut host = ComputeHost { init: &self.init, - port: &self.port, - outer: Some(outer), + primary: &mut self.primary, effort: &mut self.effort, qids: &mut queues.qids, + accounting: &mut queues.accounting, generation: self.recovery.generation(), }; - if native.is_empty() { - queue.publish(packet, dependencies, &mut host, &mut |event| deferred.push(event)) - } else { - queue.publish_with_native_waits(packet, dependencies, native, - &mut host, &mut |event| deferred.push(event)) - } + queue.publish(packet, dependencies, &mut host, &mut |event| { + deferred.push(event) + }) } } impl Registry { - /// Everything ensure_compute establishes for `owner`: the shared compute - /// objects mapped into its VM, and its queue bound to `context` with a - /// pool that fits and no spared witness to release. - fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { - let Some(queue) = self.compute(owner) else { - return Ok(false); - }; - Ok(!queue.spared_pending() - && queue.pool_fits(required) - && queue.matches(owner, queue.qid(), context)?) - } - fn release_compute_witness( &mut self, init: &super::initdata::InitData, - port: &super::port::Port, owner: u64, deferred: &mut DeferredBatch, ) -> Result { - let Some(mut queue) = self + let Some(queue) = self .compute .iter_mut() .flatten() @@ -1600,9 +951,10 @@ impl Registry { if !queue.spared_pending() { return Ok(false); } + let view = qos::View::new(init.qos()?)?; queue.release_witnessed( super::recovery::Memory::recovery_state(init)?, - |owner| port.qos_complete(&[owner]), + |owner| self.accounting.complete(&view, &[owner]), &mut |event| deferred.push(event), )?; Ok(queue.spared_pending()) @@ -1614,7 +966,7 @@ impl Registry { error: Error, deferred: &mut DeferredBatch, ) -> Result { - for mut queue in self + for queue in self .compute .iter_mut() .flatten() @@ -1641,53 +993,33 @@ impl super::Firmware { let registry = &mut self.queues; registry.observations.clear(); let mut requested = false; - // Every queue's pending identities are staged, not only the ones this batch - // retires: the scan passes each new record once, and a queue retired by a later - // batch finds its observation already latched. for queue in registry .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) { - requested |= queue.request_observations(&mut registry.observations)?; + requested |= queue.request_observations(&mut registry.observations, masks)?; } - // Completion records are appended in order: fold only the records written since - // the previous scan. A request whose stamp already covers it has its record in the - // ring; if the scan ended before finding one, read the whole ring once. if requested { - let observations = &mut registry.observations; - let seen = &mut registry.completion_seen; - registry.completion_cursor = self.init.fold_compute_completions( - registry.completion_cursor, - seen, - |record| observations.fold(record), - )?; - if observations.covered_pending() { - registry.completion_cursor = self - .init - .resync_compute_completions(seen, |record| observations.fold_any(record))?; - } + self.init + .scan_compute_completions(|record| registry.observations.fold(record))?; } - let port = &self.port; - let retired = deferred.pending(); - let mut freed = false; - for mut queue in registry + let view = qos::View::new(self.init.qos()?)?; + for queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - let busy = !queue.channel_free(); - queue.latch_observations(&mut registry.observations)?; + queue.latch_observations(&mut registry.observations, masks)?; queue.retire( masks, generation, state, - |owner| port.qos_complete(&[owner]), + |owner| registry.accounting.complete(&view, &[owner]), &mut |event| deferred.push(event), )?; - freed |= busy && queue.channel_free(); } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. @@ -1695,368 +1027,58 @@ impl super::Firmware { let terminal = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .and_then(|mut queue| queue.take_terminal()); + .and_then(compute::Queue::take_terminal); if let Some((status, error)) = terminal { status.record(error); self.settle_render_vm(&status, error, deferred)?; self.queues.fail_compute_vm(&status, error, deferred)?; } } - let port = &self.port; + let view = qos::View::new(self.init.qos()?)?; let registry = &mut self.queues; - for mut queue in registry + for queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - let busy = !queue.channel_free(); queue.release_witnessed( state, - |owner| port.qos_complete(&[owner]), + |owner| registry.accounting.complete(&view, &[owner]), &mut |event| deferred.push(event), )?; - freed |= busy && queue.channel_free(); - } - // Wake only the admission waits this batch can end. The fences gate - // scheduler entities alone; their callbacks never take the device mutex. - let shared = &self.primary.state.shared; - if freed { - shared.channels.notify(); - } - if deferred.pending() != retired { - shared.progress.notify(); } Ok(()) } } impl Backend { - fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { - admission.check()?; + fn ensure_compute(&self, scs_pages: u16) -> Result { let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; + use kernel::time::{delay::fsleep, Delta}; + const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; - loop { - admission.check()?; - let observed = self.shared.progress.observe(); + for _ in 0..SPARED_WAIT_MS { let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.replay_compute(&mut deferred)?; firmware .queues - .release_compute_witness(&firmware.init, &firmware.port, self.owner, &mut deferred) + .release_compute_witness(&firmware.init, self.owner, &mut deferred) })(); deferred.finish(); let pending = pending?; if !pending { - match self.build_compute(required, admission) { - Err(error) if error == EAGAIN => continue, - result => return result, - } - } - // Other owners continue submitting while this owner waits for its - // queue-local drain witness. No new command has been accepted; if the - // admission deadline passes, the spared queue and its retained work stay - // intact until a later admission observes its witness. - admission.extend_for_recovery(); - self.shared.queue_events(); - self.wait_release(&self.shared.progress, observed, admission)?; - } - } - - /// Steady-state publication into this owner's bound channel without the device mutex: only - /// the channel's own lock and the port's small locks are taken, and nothing device-wide is - /// scanned. None leaves the packet untouched for the device-mutex path, which owns binding, - /// first installation, input sources, render prefixes, replay and recovery. - fn compute_direct( - &self, - packet: &Arc, - required: usize, - deferred: &mut DeferredBatch, - ) -> Option { - if packet.completion.sources().is_some() - || self.context.render_seen() - || packet.cancel_requested() - || packet.completion.status().get() != 0 - || self.admission_open().is_err() - { - return None; - } - let cell = self.direct.lock().clone()?; - let lease = self.shared.preparations.try_enter().ok()?; - let port = self.shared.port(&lease)?; - if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { - return None; - } - // Full effort is applied by the device-mutex path when it is not already. - if !port.note_submission() { - return None; - } - match packet.completion.work_state().ok()?.acquire(crate::g17::job::Engine::Compute) { - Ok(_) => {} - Err(error) if error == EBUSY => return None, - Err(error) => return Some(Err(error)), - } - let mut queue = cell.lock()?; - if !queue.installed() - || queue.spared_pending() - || !queue.pool_fits(required) - || queue.pending_switch_from().is_some() - || !matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)) - || !lease.is_current() - { - return None; - } - let outer = port.reserve_compute(packet.context.policy().priority()).ok()?; - let mut host = DirectHost { - port, - outer: Some(outer), - generation: port.generation(), - }; - match queue.publish(packet.clone(), &[], &mut host, &mut |event| deferred.push(event)) { - Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, - result => Some(result), - } - } - - /// Scheduler admission of a compute packet. Shared capacity the packet lacks (a channel, a - /// work state, retirement-bound room, reopened publication) becomes a fence that the release - /// providing it signals, so publication finds it ready instead of sleeping for it. Errors - /// and anything prepare cannot judge are left to publication. - pub(super) fn prepare_compute(&self, packet: &Arc) -> Option { - if packet.early_result().is_some() - || packet.cancel_requested() - || packet.completion.status().get() != 0 - || self.admission_open().is_err() - { - return None; - } - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; - let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()).ok()?; - loop { - let channels = self.shared.channels.observe(); - let progress = self.shared.progress.observe(); - let (domain, observed) = match self.compute_readiness(packet, required) { - Readiness::Ready => return None, - Readiness::Channel => (&self.shared.channels, channels), - Readiness::Progress => (&self.shared.progress, progress), - }; - self.shared.queue_events(); - match domain.wait(observed) { - Ok(Some(fence)) => return Some(fence), - Ok(None) => continue, - Err(_) => return None, + return self.build_compute(required); } + // Other owners continue submitting while this owner gives its + // queue-local drain witness time to arrive. + fsleep(Delta::from_millis(1)); } - } - - fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { - match packet.completion.work_state().and_then(|work| work.acquire(crate::g17::job::Engine::Compute)) { - Err(error) if error == EBUSY => return Readiness::Progress, - Err(_) => return Readiness::Ready, - Ok(_) => {} - } - // The steady state: this owner's channel is bound and installed. - if packet.completion.sources().is_none() { - if let Some(cell) = self.direct.lock().clone() { - let bound = cell.lock().is_some_and(|mut queue| { - let bound = queue.installed() - && !queue.spared_pending() - && queue.pool_fits(required) - && matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)); - if bound { - queue.claim(); - } - bound - }); - if bound { - self.hold_claim(Some(cell)); - return Readiness::Ready; - } - } - } - match self.sources_published(packet) { - Ok(false) => return Readiness::Progress, - Err(_) => return Readiness::Ready, - Ok(true) => {} - } - let lease = match self.shared.preparations.try_enter() { - Ok(lease) => lease, - Err(error) if error == EAGAIN => return Readiness::Progress, - Err(_) => return Readiness::Ready, - }; - let state = self.shared.state.lock(); - let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; - if !lease.is_current() - || firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 - { - return Readiness::Progress; - } - firmware.queues.compute_readiness(self.owner, Some(&self.context), required) - } - - /// Replaces the prepared packet's channel claim; None releases it after publication, - /// cancellation or close. - pub(super) fn hold_claim(&self, cell: Option>) { - let old = core::mem::replace(&mut *self.claimed.lock(), cell); - let freed = old.as_ref().and_then(|old| old.lock()).is_some_and(|mut queue| { - let busy = !queue.channel_free(); - queue.release_claim(); - busy && queue.channel_free() - }); - if freed { - // A channel nobody prepares into any more is shareable now. - self.shared.channels.notify(); - } - } - - /// Called with the device mutex held after a successful publication: remembers the bound - /// channel for direct publication. - fn remember_direct(&self, registry: &Registry) { - let cell = registry.compute_cell(self.owner); - let mut direct = self.direct.lock(); - let current = match (&*direct, &cell) { - (Some(old), Some(new)) => Arc::ptr_eq(old, new), - (None, None) => true, - _ => false, - }; - if !current { - let old = core::mem::replace(&mut *direct, cell); - drop(direct); - drop(old); - } - } - - /// Publishes a dependency-free compute command from its submit ioctl when - /// its queue is already bound and nothing has to wait, as renders do. Any - /// wait leaves the command, unpublished, to its scheduler. - pub(super) fn try_compute_early(&self, packet: &Arc) -> bool { - if self.compute_backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { - return false; - } - if !packet.begin_publication() { - packet.completion.fail_unpublished(ECANCELED); - packet.set_early_result(Err(ECANCELED)); - return true; - } - let mut deferred = match DeferredBatch::publication() { - Ok(deferred) => deferred, - Err(error) => { - if packet.completion.status().get() != 0 { - ::fail_vm(self, error); - } - packet.completion.fail_unpublished(error); - packet.set_early_result(Err(error)); - return true; - } - }; - let result = match self.compute_direct_packet(packet, &mut deferred) { - Some(result) => result, - None => self.compute_inline(packet, &mut deferred), - }; - if matches!(result, Err(error) if error == render::FALLBACK) && !packet.is_published() { - deferred.finish(); - packet.abandon_publication(); - return false; - } - let result = self - .finish_publication(packet, result, &mut deferred) - .map(|_| ()); - packet.set_early_result(result); - true - } - - fn compute_direct_packet( - &self, - packet: &Arc, - deferred: &mut DeferredBatch, - ) -> Option { - let super::command::Validated::Compute { scratch, .. } = &packet.command else { - return Some(Err(EINVAL)); - }; - let required = match super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) { - Ok(required) => required, - Err(error) => return Some(Err(error)), - }; - self.compute_direct(packet, required, deferred) - } - - /// compute_packet's publication section without its waits: every condition - /// it would sleep or retry for returns FALLBACK instead. - fn compute_inline( - &self, - packet: &Arc, - deferred: &mut DeferredBatch, - ) -> Result { - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; - if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { - return Err(render::FALLBACK); - } - let lease = self.enter_preparation(false)?; - match packet.completion.work_state()?.acquire(crate::g17::job::Engine::Compute) { - Err(error) if error == EBUSY => return Err(render::FALLBACK), - result => { - result?; - } - } - // A lone admission keeps the original latency-sensitive probe. - // Competing publishers otherwise spin on a binding they do not own. - if !self.context.compute_binding_hint() - && self.shared.compute_demand.load(Ordering::Relaxed) > 1 - { - return Err(render::FALLBACK); - } - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() - || self.shared.crashed.load(Ordering::Acquire) - || firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0 - { - return Err(render::FALLBACK); - } - firmware.replay_compute(deferred)?; - if !firmware.init.compute_vm_cached(self.context.vm()) - || !firmware.queues.compute_bound(self.owner, &self.context, required)? - { - return Err(render::FALLBACK); - } - let dependencies = firmware.queues.render_prefix(self.owner, packet)?; - match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { - Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { - Err(render::FALLBACK) - } - result => { - if result.is_ok() { - self.remember_direct(&firmware.queues); - } - result - } - } - } - - /// A scheduled dependency may refer to a previous recovery epoch. Do not - /// acquire consumer capacity until every live source is published now. - fn sources_published(&self, packet: &super::job::Packet) -> Result { - let Some(sources) = packet.completion.sources() else { return Ok(true); }; - sources.check()?; - let state = self.shared.state.lock(); - let firmware = state.as_deref().ok_or(ENODEV)?; - if firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0 - { return Ok(false); } - for source in &sources.sources { - if source.submission.source_status()? { continue; } - if !firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.has_input_source(&source.submission, - packet.completion.status(), firmware.recovery.generation())) - { return Ok(false); } - } - Ok(true) + // No new command has been accepted. Keep the spared queue and its + // retained work intact until a later admission observes its witness. + Err(EBUSY) } /// Called after taking this packet's publication claim. All waits and work @@ -2067,70 +1089,23 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; - let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; - if let Some(result) = self.compute_direct(packet, required, deferred) { - return result; - } - // A bound queue needs none of ensure_compute's admission passes. Try - // without them; the publication section below checks the same state. - // Every retry blocks on the release that ends its wait: retirement - // progress (work states, ring room, replays) and the reopening of - // publication after recovery both signal `progress`. - let mut bound = true; - let mut demand = None; - let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); - let check = || { - if packet.cancel_requested() { return Err(ECANCELED); } - self.admission_open()?; - if packet.completion.status().get() != 0 { return Err(EIO); } - Ok(()) - }; - let admission = Admission::new(deadline, &check); - let progress = &self.shared.progress; + let mut recovery_polls = 0; + let mut outer_wait = None; loop { - admission.check()?; - let observed = progress.observe(); - if !self.sources_published(packet)? { - let recovering = self.shared.state.lock().as_deref().is_some_and(|firmware| { - firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 - }); - if recovering { - admission.extend_for_recovery(); - } - self.shared.queue_events(); - self.wait_release(progress, observed, &admission)?; - continue; - } - // All DRM dependencies were satisfied before this scheduler callback. - // Recheck on every retry: pressure may yield an idle binding between - // admission and publication without changing this logical queue. - if !bound { - demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); - Backend::ensure_compute(self, scratch.pages(), &admission)?; - admission.check()?; - } - let lease = self.shared.preparations.enter_until(&admission)?; - // Work states free as this context's earlier commands retire. - match packet.completion.work_state()?.acquire(crate::g17::job::Engine::Compute) { - Err(error) if error == EBUSY => { - drop(lease); - self.shared.queue_events(); - self.wait_release(progress, observed, &admission)?; - continue; - } - result => { - result?; - } + if packet.completion.status().get() != 0 { + return Err(EIO); } + let lease = self.shared.preparations.enter(false)?; + packet.completion.work_state()?.acquire()?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { drop(state); continue; } - admission.check()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } if self .shared .crashed @@ -2140,51 +1115,52 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { - admission.extend_for_recovery(); + if recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(EBUSY); + } + recovery_polls += 1; drop(state); drop(lease); self.shared.queue_events(); - self.wait_release(progress, observed, &admission)?; + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); continue; } firmware.replay_compute(deferred)?; - if !firmware.init.compute_vm_cached(self.context.vm()) - || !firmware.queues.compute_bound(self.owner, &self.context, required)? - { - bound = false; - drop(state); - drop(lease); - continue; - } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; - admission.check()?; let result = firmware.publish_compute( self.owner, packet.clone(), dependencies.as_slice(), deferred, ); - if !packet.is_published() - && (matches!(result, Err(EAGAIN)) - || (matches!(result, Err(EBUSY)) - && (firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0))) + if matches!(result, Err(EBUSY) | Err(EAGAIN)) + && !packet.is_published() + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0) { - // Recovery or backpressure: nothing is firmware-visible. Other - // queues keep submitting and the event worker retires meanwhile. - if firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0 - { - admission.extend_for_recovery(); + if recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(EBUSY); } + recovery_polls += 1; drop(state); drop(lease); self.shared.queue_events(); - self.wait_release(progress, observed, &admission)?; + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); continue; } - if result.is_ok() { - self.remember_direct(&firmware.queues); + if matches!(result, Err(EAGAIN)) && !packet.is_published() { + let now = now_ns(); + let since = *outer_wait.get_or_insert(now); + if now.saturating_sub(since) >= WORK_RING_WAIT_NS { + return Err(EBUSY); + } + // Backpressure owns no firmware-visible attempt. Other queues + // can submit and the event worker can retire while we sleep. + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; } return result; } @@ -2197,7 +1173,7 @@ impl super::Firmware { /// or an allocation-dependent collector. fn finish_stopped_queues(&mut self) { self.stop_render(); - for mut queue in self + for queue in self .queues .compute .iter_mut() @@ -2223,41 +1199,6 @@ impl super::Firmware { } impl super::Shared { - /// Charge only real watchdog expirations, after the per-job allowance. A - /// changing client or recovery generation cannot mint a fresh storm budget. - pub(super) fn watchdog_timeout(self: &Arc) -> bool { - if self.crashed.load(Ordering::Acquire) { return true; } - let stop = { - let mut state = self.state.lock(); - state.as_deref_mut().is_none_or(|firmware| firmware.recovery.timeout_limit(now_ns())) - }; - if stop { self.lose_device_quietly(); } - stop - } - - /// Workload-triggered stop: halt producers first, then join only for a - /// bounded interval. A stalled borrower leaves stopped firmware and all its - /// mappings owned in state. Only a later joined stop may detach/free them. - pub(super) fn stop_runtime(&self) { - let Some(_stopping) = self.stop.try_lock() else { return; }; - { - let mut state = self.state.lock(); - if let Some(firmware) = state.as_deref_mut() { - if let Some(secondary) = firmware.secondary.as_mut() { - secondary.stop(); - } - firmware.primary.stop(); - } - } - if self.preparations.wait_drained_until(now_ns().saturating_add(WORK_RING_WAIT_NS)).is_err() { - dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); - return; - } - self.forget_port(); - let firmware = self.state.lock().take(); - if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } - } - /// Joins construction and complete host teardown, including RTKit drops. /// The stop-only mutex outlives detached firmware destruction, so unbind /// cannot return while another stop caller still owns mailbox clients. @@ -2267,13 +1208,10 @@ impl super::Shared { pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); let _stopping = self.stop.lock(); - self.forget_port(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { - if let Some(secondary) = firmware.secondary.as_mut() { - secondary.stop(); - } + firmware.secondary.stop(); firmware.primary.stop(); } state.take() @@ -2307,63 +1245,26 @@ impl super::Firmware { deferred: &mut DeferredBatch, now: u64, ) -> Result> { - let mut freed = false; - let mut releasing = false; - for mut queue in self + for queue in self .queues .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - let busy = !queue.channel_free(); queue.recycle_killed()?; queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } - freed |= busy && queue.channel_free(); - releasing |= queue.pool_release_pending(); - } - freed |= self.queues.observe_pool_releases(&self.init, deferred)? != 0; - let shared = &self.primary.state.shared; - // Edge notifications cover the transitions this driver makes; every - // event pass also rechecks the level, so capacity that appeared through - // another engine (render QIDs, closed logical queues, render-held work - // states) still ends a scheduler wait that has no timeout of its own. - if freed - || (shared.channels.has_waiters() - && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) - { - shared.channels.notify(); - } - // A pool release has no completion event: each observation pass is - // the progress a growing owner waits for. - if releasing || shared.progress.has_waiters() { - shared.progress.notify(); - } - if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.pool_release_stalled(now)) { - return Err(ETIMEDOUT); } + self.queues.observe_pool_releases(&self.init, deferred)?; if self.render_control_backpressured() { return Ok(None); } - // While the shared lower window refuses free-list backing, no idle - // retained compute pool stays warm: render and new compute need it more. - let cap = if super::freelist::lower_pressure() { - 0 - } else if self.primary.state.shared.compute_demand.load(Ordering::Acquire) != 0 { - // A retired channel being requested by another runnable publisher - // is not an unused warm pool. Keep its existing backing through - // handoff; real allocation pressure above still forces reclamation. - super::kick::COMPUTE_CHANNELS - } else { - COMPUTE_WARM_CAP - }; let (published, next) = self.queues.vacate_idle_pools( &self.init, - cap, + COMPUTE_WARM_CAP, now, COMPUTE_WARM_EXPIRY_NS, )?; @@ -2375,220 +1276,3 @@ impl super::Firmware { } } - -impl Registry { - pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - self.qids.write_capacity(f)?; - self.admission.write_capacity(f)?; - self.buffers.write_capacity(f)?; - writeln!(f, "qos_ids used={} total=126", self.qos_ids.used())?; - writeln!(f, "metrics_ids used={} total={}", self.metrics.used(), super::fw::buffer::METRICS_SLOTS)?; - self.render.write_capacity(f)?; - let now = now_ns(); - let waits = || self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) - .filter_map(|queue| queue.kill_waiting(now)); - compute_exit::write_kill_waits(f, waits().count(), waits().max().unwrap_or(0))?; - compute_exit::write_render_kills(f, self.teardown.kill_waits(now).count(), - self.teardown.kill_waits(now).max().unwrap_or(0))?; - writeln!(f, "compute_slots used={} total={} reusable={} building={}", - self.compute.iter().flatten().count(), self.compute.len(), - self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), - self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; - writeln!(f, "compute_channels used={} fresh_fits={}", - self.compute.iter().flatten().count(), - u8::from(self.qids.compute_fits(self.admission.queues() as usize)))?; - writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; - for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { - queue.write_capacity(f)?; - } - Ok(()) - } -} - -impl Backend { - /// Compute admission without a channel: give back what pressure allows, then block until a - /// channel is released after `observed` (sampled before the failed check). - fn wait_channels(&self, observed: u64, admission: &Admission<'_>, reclaim_backing: bool) -> Result { - self.reclaim_capacity(admission, reclaim_backing)?; - self.wait_release(&self.shared.channels, observed, admission) - } - - /// Blocks until `domain` is signalled after `observed`, this packet's admission ends, or one - /// admission slice passes: cancellation and logical close have no release to wait for. - fn wait_release( - &self, - domain: &super::availability::Availability, - observed: u64, - admission: &Admission<'_>, - ) -> Result { - use kernel::dma_fence::RawDmaFence; - let slice = admission.slice_ms()?; - if let Some(fence) = domain.wait(observed)? { - // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. - let waited = unsafe { - kernel::bindings::dma_fence_wait_timeout( - fence.raw(), - false, - kernel::time::msecs_to_jiffies(slice) as isize, - ) - }; - if waited < 0 { - return Err(Error::from_errno(waited as i32)); - } - } - admission.check() - } - - /// Reclaim steps of a capacity wait that never sleep: observe consumed pool releases and, - /// under backing pressure, give idle pools back. - fn reclaim_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { - admission.check()?; - let mut deferred = DeferredBatch::publication()?; - let result: Result = (|| { - // A short reclaim transaction obeys the same session/epoch gate as - // publication. Never keep this lease while sleeping for capacity. - let lease = self.shared.preparations.try_enter()?; - let mut state = self.shared.state.lock(); - let firmware = state.as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { - return Err(EAGAIN); - } - if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { - return Ok(()); - } - // Consumption, not publication or an elapsed timeout, frees backing. - firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; - // Pool eviction addresses backing allocation pressure, not channel - // occupancy. Normal idle expiry and the retained-pool cap still run - // in the event worker for either kind of admission wait. - if reclaim_backing { - firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; - } - if reclaim_backing && !firmware.render_control_backpressured() { - let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; - if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } - } - Ok(()) - })(); - deferred.finish(); - self.shared.queue_events(); - match result { - Err(error) if error == EAGAIN => {} - result => result?, - } - admission.check() - } -} - -#[kunit_tests(asahi_g17_shared_capacity)] -mod shared_capacity_tests { - use super::*; - use crate::g17::fw::render::Deflake; - use crate::g17::fw::variant::V26A428; - - #[test] - fn all_logical_render_owners_fit_beside_shared_compute() { - let mut registry = KBox::init( - Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), - GFP_KERNEL, - ) - .unwrap(); - let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - let mut slots = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - for _ in 0..crate::mmu::MAX_EXECUTION_CONTEXTS { - slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); - } - // With every execution context open, compute keeps only the channels - // that leave each logical queue a dedicated render pair. - for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { - registry.reserve_compute(owner).unwrap(); - } - assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); - assert!(registry.qids.render_fits(slots.len())); - for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); - registry.ensure_render_ids(lease.slot(), owner).unwrap(); - leases.push(lease, GFP_KERNEL).unwrap(); - } - assert_eq!(leases.len(), crate::mmu::MAX_EXECUTION_CONTEXTS as usize); - // Every accepted logical render owner now has its physical pair; no - // published identity was cancelled or retyped to achieve the fit. - } - - #[test] - fn a_qid_floor_admits_only_the_render_pairs_it_leaves() { - use crate::g17::fw::variant::V25G83; - let limit = super::super::kick::queue_limit(V25G83.qid_floor); - assert_eq!(limit, 42); - let mut registry = KBox::init( - Registry::new(2, Deflake::TWO_CLUSTERS, [0; 2], &V25G83), - GFP_KERNEL, - ) - .unwrap(); - let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); - let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); - for owner in 1..=u64::from(limit) { - slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); - registry.ensure_render_ids(lease.slot(), owner).unwrap(); - leases.push(lease, GFP_KERNEL).unwrap(); - } - // The next logical queue is refused when it is created, not left waiting for a pair. - assert!(matches!(registry.admission.reserve_queue(), Err(ENOSPC))); - } - - #[test] - fn four_clients_have_capacity_before_any_owner_closes() { - let mut registry = KBox::init( - Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), - GFP_KERNEL, - ) - .unwrap(); - let mut leases = KVec::with_capacity(36, GFP_KERNEL).unwrap(); - for client in 0..4u64 { - for queue in 0..9u64 { - let owner = 1 + client * 9 + queue; - let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); - registry.ensure_render_ids(lease.slot(), owner).unwrap(); - leases.push(lease, GFP_KERNEL).unwrap(); - } - } - for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { - registry.reserve_compute(owner).unwrap(); - } - assert_eq!(leases.len(), 36); - } - - #[test] - fn compute_grows_with_free_qids_and_narrows_queue_admission() { - let mut registry = KBox::init(Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), GFP_KERNEL).unwrap(); - let mut slots = KVec::with_capacity(20, GFP_KERNEL).unwrap(); - for _ in 0..20 { - slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); - } - // Sixteen open compute clients each keep their own channel. - for owner in 1..=16u64 { - registry.reserve_compute(owner).unwrap(); - } - // Growth stops where open queues plus headroom would lose their pairs. - let mut owner = 17; - while registry.reserve_compute(owner).is_ok() { - owner += 1; - } - assert!(registry.qids.render_fits(slots.len() + 24)); - assert!(!registry.qids.render_fits(crate::mmu::MAX_EXECUTION_CONTEXTS as usize)); - } -} - -impl Backend { - /// Logical close is independent of whether an off-lock builder has already - /// installed a physical graph. No constructor can resurrect a closed owner. - fn admission_open(&self) -> Result { - if self.closed.load(Ordering::Acquire) { return Err(ECANCELED); } - if self.context.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { - return Err(EIO); - } - if !self.context.is_current() { return Err(EFAULT); } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index bf0f989f042a5..3d5f441d8428f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -13,52 +13,13 @@ use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { if packet.take_backlog() { - self.backlog_of(packet).fetch_sub(1, Ordering::AcqRel); - } - } - - pub(super) fn backlog_of(&self, packet: &Packet) -> &core::sync::atomic::AtomicU32 { - match packet.engine() { - Engine::Render => &self.backlog, - Engine::Compute => &self.compute_backlog, + self.backlog.fetch_sub(1, Ordering::AcqRel); } } fn stop_after_error(&self) { - // Scheduler/fence callbacks never synchronously join builders or RTKit - // receive work. The event worker performs the bounded runtime stop. self.shared.lose_device_quietly(); - } - - fn rollback_unpublished_compute(&self) -> Result { - // No outer preparation lease survives compute_packet's return. Hold a - // fresh epoch through alias destruction so recovery can join this work. - loop { - let lease = self.shared.preparations.enter(false)?; - let mut retired = None; - let mut state = self.shared.state.lock(); - let firmware = state.as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { drop(state); drop(lease); continue; } - let mut binding = None; - if let Some(index) = firmware.queues.compute.iter().position(|entry| { - entry.as_ref().and_then(|entry| entry.queue.as_deref()) - .is_some_and(|queue| queue.owner() == Some(self.owner)) - }) { - let entry = firmware.queues.compute[index].as_mut().ok_or(EIO)?; - let (cancel, replaced) = entry.queue.as_deref_mut().ok_or(EIO)? - .rollback_unpublished(self.owner, &self.context)?; - binding = replaced; - if cancel { - retired = entry.queue.take().map(|queue| (entry.reservation, queue)); - } - } - drop(state); - drop(binding); - self.finish_uninstalled_compute(retired)?; - self.shared.channels.notify(); - self.shared.queue_events(); - return Ok(()); - } + self.shared.stop_queues(); } /// Complete all failure fanout before signalling this packet's unpublished @@ -75,11 +36,6 @@ impl Backend { } deferred.finish(); if !packet.is_published() { - if packet.engine() == Engine::Compute { - if self.rollback_unpublished_compute().is_err() { - self.stop_after_error(); - } - } packet.completion.fail_unpublished(error); } Err(error) @@ -90,10 +46,6 @@ impl Backend { } fn release_queue(&self) -> Result { - // A builder may own only an off-lock reservation when close arrives. - // Record intent before scanning queues; every later install/rearm checks it. - self.closed.store(true, Ordering::Release); - self.hold_claim(None); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -108,63 +60,12 @@ impl Backend { firmware.queues.release_render(self.owner)?; firmware.queues.admission.release_owner(self.owner); drop(state); - // Its idle channel, or its slot and never-published QID, is free. - self.shared.channels.notify(); self.shared.queue_events(); Ok(()) } } impl job::Backend for Backend { - fn input_sources(&self, inputs: &[crate::file::SyncItem]) - -> Option> - { - use crate::g17::{fence, sync_input}; - let mut heads = KVec::new(); - let mut sources: KVec = KVec::new(); - let mut publication_waits = KVec::new(); - for input in inputs.iter().filter_map(|input| input.fence.as_ref()) { - let status = job::fence_status(input); - if status > 0 { continue; } - if status < 0 { return None; } - let leaves = sync_input::collect(input)?; - for leaf in &leaves { - if leaf.status > 0 { continue; } - if leaf.status < 0 { return None; } - // SAFETY: Collection owns each leaf until this lookup finishes. - let submission = unsafe { fence::exported_source_raw(leaf.fence) }?; - if submission.source_status().ok()? { continue; } - if !submission.same_vm(self.context.status()) { return None; } - if sources.iter().any(|known| Arc::ptr_eq(&known.submission, &submission)) { - continue; - } - if sources.len() == crate::g17::fw::kick::KICK_BARRIERS_MAX { return None; } - let published = { - let state = self.shared.state.lock(); - let firmware = state.as_deref()?; - if self.shared.crashed.load(Ordering::Acquire) || firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init).ok()? != 0 - { return None; } - firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.has_input_source(&submission, self.context.status(), - firmware.recovery.generation())) - }; - if !published { - if submission.source_status().ok()? { continue; } - // The existing scheduler fence becomes ready after actual - // publication, not GPU completion. No binding loan or - // consumer hardware capacity is acquired while waiting. - publication_waits.push(submission.ready()?, GFP_KERNEL).ok()?; - } - sources.push(sync_input::Source { submission }, GFP_KERNEL).ok()?; - } - heads.push(input.clone(), GFP_KERNEL).ok()?; - } - if sources.is_empty() { return None; } - sync_input::Sources::new(heads, sources, publication_waits).ok() - } - fn feed(&self) -> Option> { Some(self.shared.feed.clone()) } @@ -174,11 +75,7 @@ impl job::Backend for Backend { } fn ensure_compute(&self, scs_pages: u16) -> Result { - // A reverse timeline chain must be submitted before the host signal. - // Validate here, but do not consume physical capacity behind unsignalled - // DRM dependencies. compute_packet admits it when the job can run. - crate::g17::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages) - .map(|_| ()) + Backend::ensure_compute(self, scs_pages) } fn fail_vm(&self, error: Error) { @@ -216,13 +113,8 @@ impl job::Backend for Backend { /// Firmware kick parents order the same engine. Explicit cross-engine and /// input dependencies were already attached to the DRM scheduled job. - /// Compute and render admission wait for their shared capacity here, as a - /// fence. - fn prepare(&self, packet: &Arc) -> Option { - match packet.engine() { - Engine::Compute => self.prepare_compute(packet), - Engine::Render => self.admit_render(packet), - } + fn prepare(&self, _packet: &Packet) -> Option { + None } fn publish(&self, packet: Arc) -> Result { @@ -250,25 +142,12 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; - if packet.engine() == Engine::Compute { - self.hold_claim(None); - } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } fn timed_out(&self, packet: Arc) -> sched::Status { - // A packet still in publication, for example waiting for capacity or - // for its render pair's graph, has nothing visible to firmware and may - // not even have an installed pair. Its publisher owns its bounded wait - // and result, and it keeps its place in the backlog so that a later - // dependency-free command is not published from its ioctl ahead of it. - // It is not a hang and does not count towards the timeout storm limit. - if !packet.is_published() { - return sched::Status::NoHang; - } self.release_backlog(&packet); - if self.shared.watchdog_timeout() { return sched::Status::NoHang; } let result = match packet.engine() { Engine::Compute => self.timeout_compute(&packet), Engine::Render => self.timeout_render(&packet), @@ -281,9 +160,6 @@ impl job::Backend for Backend { fn cancel(&self, packet: Arc) { self.release_backlog(&packet); - if packet.engine() == Engine::Compute { - self.hold_claim(None); - } if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; @@ -297,72 +173,22 @@ impl job::Backend for Backend { } } - fn long_job_beside(&self, _packet: &Arc) -> bool { - let state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref() else { - return false; - }; - firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|other| other.owner() != Some(self.owner) && other.long_job_running()) - } - fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool { if packet.engine() == Engine::Compute { return self.defer_compute_timeout(packet, renewals); } - use crate::g17::timeout::{self, Decision}; - // As for compute, a final retirement pass precedes the attribution, and all completion - // callbacks run after releasing the device mutex. Without a collector the attribution - // goes ahead without that pass, as it did before it existed. - let mut deferred = DeferredBatch::worker().ok(); - let result = (|| -> Result { - let mut state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(false); - }; - if let Some(deferred) = deferred.as_mut() { - firmware.scan_renders(deferred)?; - } - // Borrowed-graph and queued-retirement deferrals share the renewal bound. - if timeout::transaction_grace(renewals) - && firmware.queues.defer_render_timeout(packet, ETIMEDOUT) - { - return Ok(true); - } - let verdict = firmware.queues.render_timeout_decision(packet, renewals); - let decision = verdict.map_or(Decision::Own, |verdict| verdict.decision); - // As compute does: work still blocked behind another VM's older unfinished work - // after the whole grace is innocent, so its failure must stay non-sticky. The pair's - // quarantine extends the class to the pair's other groups. - if decision == Decision::Spare { - packet.completion.classify(crate::g17::recovery::Class::Spared); - } - if let Some(verdict) = verdict { - if let Some(blocker) = verdict.reported_blocker(renewals) { - dev_warn!(self.shared.dev.as_ref(), - "Render watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", - self.owner, verdict.own.vm, verdict.own.published, renewals, decision, - blocker.vm, blocker.published, u8::from(blocker.failed)); - } - } - Ok(decision == Decision::Defer) - })(); - if let Some(deferred) = deferred.as_mut() { - deferred.finish(); - } - matches!(result, Ok(true)) + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .is_some_and(|firmware| firmware.queues.defer_render_timeout(packet, ETIMEDOUT)) } fn try_early(&self, packet: &Arc) -> bool { - match packet.engine() { - Engine::Render => self.try_render_early(packet), - Engine::Compute => self.try_compute_early(packet), - } + self.try_render_early(packet) } fn count_backlog(&self, packet: &Packet) { packet.count_backlog(); - self.backlog_of(packet).fetch_add(1, Ordering::AcqRel); + self.backlog.fetch_add(1, Ordering::AcqRel); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 4f519743aed53..dac4ca48a34ae 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,28 +20,26 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; - // Each queue is locked on its own: release this one before visiting the others. - let Some(own) = firmware.queues.compute(self.owner) - .filter(|queue| queue.can_classify_timeout(packet)) - .and_then(|queue| queue.timeout_head()) else { + let Some(queue) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) else { return Ok(Decision::Own); }; - let computes = firmware.queues.compute.iter().flatten() + let Some(own) = queue.timeout_head() else { + return Ok(Decision::Own); + }; + let others = firmware.queues.compute.iter().flatten() .filter_map(|entry| entry.queue.as_deref()) .filter(|other| other.owner() != Some(self.owner)) .filter_map(|other| other.timeout_head()); - let others = Iterator::chain(computes, firmware.queues.render_timeout_heads()); - let verdict = timeout::attribute(own, renewals, others); - let decision = verdict.decision; + let decision = timeout::classify(own, renewals, others); if decision == Decision::Spare { firmware.queues.compute(self.owner).ok_or(EIO)? .classify(&|_| Class::Spared); } - if let Some(blocker) = verdict.reported_blocker(renewals) { + if decision != Decision::Own && (renewals == 0 || decision == Decision::Spare) { dev_warn!(self.shared.dev.as_ref(), - "Compute watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", - self.owner, own.vm, own.published, renewals, decision, - blocker.vm, blocker.published, u8::from(blocker.failed)); + "Compute watchdog owner={} publication={} renewals={} decision={:?} behind earlier work of another VM\n", + self.owner, own.published, renewals, decision); } Ok(decision) })(); @@ -52,10 +50,7 @@ impl Backend { /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - // Without a collector the watchdog expires again and retries. - let Ok(mut deferred) = DeferredBatch::worker() else { - return Ok(()); - }; + let mut deferred = DeferredBatch::worker()?; let result = (|| { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -63,7 +58,7 @@ impl Backend { return Ok(None); }; firmware.scan_compute(None, &mut deferred)?; - let Some(mut queue) = firmware.queues.compute_mut(self.owner) else { + let Some(queue) = firmware.queues.compute_mut(self.owner) else { return Ok(None); }; if !queue.owns_packet(packet) { @@ -90,7 +85,7 @@ impl Backend { return Ok(()); }; firmware.scan_compute(None, &mut deferred)?; - if let Some(mut queue) = firmware.queues.compute_mut(self.owner) { + if let Some(queue) = firmware.queues.compute_mut(self.owner) { queue.cancel(packet, &mut |event| deferred.push(event))?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 32b9acef94c0f..0d4a29e779b4a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -3,7 +3,7 @@ //! Ordered republication of witnessed, unexecuted compute commands. use super::DeferredBatch; -use crate::g17::{completion::Deferred, recovery, Firmware}; +use crate::g17::{completion::Deferred, qos, recovery, Firmware}; use kernel::prelude::*; impl Firmware { @@ -31,12 +31,12 @@ impl Firmware { return Ok(()); } deferred.reserve_replays(commands)?; - let port = &self.port; + let view = qos::View::new(self.init.qos()?)?; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(mut queue) = entry.queue.as_deref_mut() { + if let Some(queue) = entry.queue.as_deref_mut() { queue.prepare_replays( state, - |owner| port.qos_complete(&[owner]), + |owner| self.queues.accounting.complete(&view, &[owner]), &mut |event| deferred.push(event), )?; } @@ -46,14 +46,20 @@ impl Firmware { if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { return Ok(()); } - let Some((packet, owner)) = self.queues.compute[index] + let Some(queue) = self.queues.compute[index] .as_ref() .and_then(|entry| entry.queue.as_deref()) - .and_then(|queue| Some((queue.replay_front()?, queue.owner()))) else { break; }; - let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; + let Some(packet) = queue.replay_front() else { + break; + }; + let owner = queue.owner(); + let outer_ready = self.init.work_ready( + packet.context.policy().priority(), + crate::g17::fw::queue::DataMaster::Compute, + )?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); @@ -77,14 +83,14 @@ impl Firmware { .ok_or(EIO)? .prepare_replay(&packet)?; let owner = owner.ok_or(EIO)?; - packet.check_replay_dependencies()?; + packet.check_dependencies()?; let dependencies = self.queues.render_prefix(owner, &packet)?; self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) })() }; let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); - let mut queue = self.queues.compute[index] + let queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; @@ -104,8 +110,6 @@ impl Firmware { } } self.queues.replay_outer_wait = None; - // Publication behind these replays may proceed. - self.primary.state.shared.progress.notify(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3e779bba4fbbf..3ae1d0a609a33 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -15,9 +15,6 @@ use kernel::{prelude::*, sync::Arc}; /// Wake interval while a tiler heap may still shrink: the native check period. const SHRINK_WAKE_NS: u64 = 1_000_000_000; -/// Longest interval between two event passes that run every housekeeping step while -/// completion records alone keep the worker busy. -const HOUSEKEEPING_NS: u64 = 1_000_000; struct Service<'a> { firmware: &'a mut Firmware, @@ -27,30 +24,24 @@ struct Service<'a> { } impl event::Transport for Service<'_> { - fn roles(&self) -> &'static [Role] { - self.firmware.roles() - } fn trace(&mut self, role: Role) { - let firmware = &mut *self.firmware; + let firmware = &mut self.firmware; firmware .rings .drain_trace(&firmware.primary.state.shared.dev, &firmware.init, role); } fn statistics(&mut self, role: Role) { - let firmware = &mut *self.firmware; - firmware.observe_statistics_power(); + let firmware = &mut self.firmware; firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); - firmware.observe_statistics_power(); } fn drain( &mut self, role: Role, mut consume: impl FnMut(&mut Self, Event) -> Result, ) -> Result { - let mut events = - channel::Events::new(role, self.firmware.primary.state.shared.variant.graph); + let mut events = channel::Events::new(role); while let Some(event) = events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? { @@ -78,15 +69,6 @@ impl event::Queues for Service<'_> { fn context_killed(&mut self, cookie: u64) { self.firmware.queues.acknowledge_compute_exit(cookie); } - fn compute_fault(&mut self, qid: u8, stamp: u32) { - for entry in self.firmware.queues.compute.iter_mut().flatten() { - if let Some(mut queue) = entry.queue.as_deref_mut() { - if queue.qid() == qid { - queue.note_fault(stamp); - } - } - } - } fn flush_tvb_reply(&mut self) -> Result { self.firmware.flush_tvb_reply() } @@ -146,8 +128,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entry(&self, index: usize) -> Result<(u32, u32)> { - self.firmware.init.entry(index) + fn entries(&self) -> Result { + self.firmware.init.entries() } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -218,27 +200,6 @@ impl recovery::Queues for Service<'_> { } self.firmware.queues.classify_render(class) } - fn note_interrupted(&mut self, qid: u8, stamp: u32) -> recovery::Interrupted { - let render = self.firmware.queues.note_render_interrupted(qid, stamp); - if render != recovery::Interrupted::None { - return render; - } - self.firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .find(|queue| queue.qid() == qid) - .map_or(recovery::Interrupted::None, |queue| { - queue.note_interrupted(stamp) - }) - } - fn report_blamed_render(&self, qid: u8) { - self.firmware - .queues - .report_render_recovery(qid, &self.firmware.port); - } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self .firmware @@ -261,17 +222,19 @@ impl recovery::Queues for Service<'_> { impl Firmware { fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { let state = recovery::Memory::recovery_state(&self.init)?; - let port = &self.port; + let view = crate::g17::qos::View::new(self.init.qos()?)?; let registry = &mut self.queues; - for mut queue in registry + for queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.release_witnessed(state, |owner| port.qos_complete(&[owner]), &mut |event| { - deferred.push(event) - })?; + queue.release_witnessed( + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; } Ok(()) } @@ -300,11 +263,6 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { - self.rings.invalidate_statistics(); - if self.recovery.restart_limit(super::now_ns()) { - pr_warn!("G17: restart storm limit reached\n"); - return Err(ETIMEDOUT); - } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, @@ -314,15 +272,7 @@ impl Firmware { }; let result = recovery::service(&mut service, || { shared.crashed.load(core::sync::atomic::Ordering::Acquire) - }); - // Publishers without the device mutex stamp their tickets with this - // mirror; they rejoin only after preparation admission reopens. - service - .firmware - .port - .set_generation(service.firmware.recovery.generation()); - service.firmware.rings.invalidate_statistics(); - let result = result?; + })?; if let Some(error) = result.target_error { dev_warn!( shared.dev.as_ref(), @@ -350,7 +300,7 @@ impl crate::g17::Shared { /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { if self.crashed.load(core::sync::atomic::Ordering::Acquire) { - self.stop_runtime(); + self.stop_queues(); return; } let mut deferred = { @@ -365,10 +315,8 @@ impl crate::g17::Shared { }; deferred }; - crate::g17::trace::event(0); let result = self.runtime_event_turn(&mut deferred); deferred.finish(); - crate::g17::trace::event(2); self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); @@ -376,15 +324,11 @@ impl crate::g17::Shared { if let Err(error) = result { dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); self.lose_device_quietly(); - self.stop_runtime(); + self.stop_queues(); } } fn runtime_event_turn(self: &Arc, deferred: &mut DeferredBatch) -> Result { - // Taken before consuming records: a request made during this pass gets the next one. - let requested = self - .housekeeping - .swap(false, core::sync::atomic::Ordering::AcqRel); let (recover, masks) = { let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -400,10 +344,8 @@ impl crate::g17::Shared { (recover, masks) }; deferred.finish(); - crate::g17::trace::event(1); if recover { - self.preparations - .wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; + self.preparations.wait_drained(); } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -446,44 +388,19 @@ impl crate::g17::Shared { .any(|queue| queue.in_flight()); let in_flight = compute || firmware.queues.render_in_flight(); let idle = firmware.idle_check(now, in_flight)?; - // Expiries, drained pairs, exits, pool and heap trimming and teardown give back what - // retirement freed; a completion needs none of them to retire or to signal. While work - // is in flight and nothing waits for capacity, a pass for firmware records alone skips - // them. Any requested pass, recovery, teardown, a waiter, the last retirement and, at - // the latest, the first pass after HOUSEKEEPING_NS run them all. - let housekeeping = requested - || recover - || !in_flight - || firmware.recovery.pending() - || !firmware.queues.teardown.is_empty() - || self.channels.has_waiters() - || self.progress.has_waiters() - || self.queues.has_waiters() - || self.renders.has_waiters() - || now.saturating_sub( - self.housekeeping_ns - .load(core::sync::atomic::Ordering::Relaxed), - ) >= HOUSEKEEPING_NS; - let mut lingers = None; - let mut pools = None; + firmware.reclaim_drained_renders(deferred)?; + firmware.service_compute_exit(deferred)?; + let pools = firmware.service_compute_pools(deferred, now)?; let mut shrinking = false; - if housekeeping { - self.housekeeping_ns - .store(now, core::sync::atomic::Ordering::Relaxed); - if !firmware.recovery.pending() { - lingers = firmware.expire_render_lingers(now)?; - } - firmware.reclaim_drained_renders(deferred)?; - firmware.service_compute_exit(deferred)?; - pools = firmware.service_compute_pools(deferred, now)?; - if !firmware.recovery.pending() { - firmware - .queues - .vacate_evicted_render_pools(&firmware.init, deferred)?; - shrinking = firmware - .queues - .shrink_render_heaps(&firmware.init, now, deferred); - } + if !firmware.recovery.pending() { + firmware + .queues + .vacate_evicted_render_pools(&firmware.init, deferred)?; + shrinking = firmware + .queues + .shrink_render_heaps(&firmware.init, now, deferred); + } + { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; if !reclaim.is_empty() { @@ -502,7 +419,7 @@ impl crate::g17::Shared { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); + .any(|queue| queue.retirement_polling()); if polling { self.queue_poll(); } @@ -513,14 +430,11 @@ impl crate::g17::Shared { } else { idle }; - // A retained compute pool gives its backing back when it expires, and - // so does an idle render free-list registration. - let idle = [pools, lingers] - .into_iter() - .fold(idle, |idle, expiry| match (idle, expiry) { - (Some(delay), Some(expiry)) => Some(delay.min(expiry)), - (delay, expiry) => delay.or(expiry), - }); + // A retained compute pool gives its backing back when it expires. + let idle = match (idle, pools) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }; if let Some(delay) = idle { self.queue_idle(delay); } @@ -530,23 +444,17 @@ impl crate::g17::Shared { } else if !recover && !polling && (channel::Events::pending(&firmware.init, Role::Primary)? - || (firmware.secondary.is_some() - && channel::Events::pending(&firmware.init, Role::Secondary)?)) + || channel::Events::pending(&firmware.init, Role::Secondary)?) { // A full bounded pass must not depend on a fresh notification for // records that arrived while it was consuming the same ring. - self.queue_event_records(); + self.queue_events(); } let grow_ready = firmware.render_grow_ready(); drop(state); if grow_ready { self.queue_grow(); } - // Firmware ring room, consumed releases and teardown steps have no - // event of their own: recheck render publishers' conditions every pass. - if self.renders.has_waiters() { - self.renders.notify(); - } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 6657f83e9e738..a47abf2045526 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -6,7 +6,6 @@ mod exit; mod dependencies; -mod direct; mod grow; mod observe; mod pools; @@ -17,7 +16,6 @@ mod settle; mod submit; pub(super) mod teardown; -pub(super) use submit::FALLBACK; use super::{DeferredBatch, Registry}; use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; use crate::hw::t8140::queues::RENDER_SLOTS; @@ -26,7 +24,6 @@ use kernel::{prelude::*, sync::Arc}; #[derive(Copy, Clone)] struct Config { clusters: u32, - deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], } @@ -49,13 +46,7 @@ struct Entry { pending_owner_closed: bool, borrowed_active: bool, borrowed_dependencies: Option, - /// `Pair::switch_in_flight` of a borrowed pair. A borrowed pair publishes - /// nothing, so the snapshot can only be stale towards an older switch. - borrowed_switch: Option, event_masks: [u64; 2], - /// A completion event for this pair arrived while it was borrowed. Event - /// passes skip borrowed pairs, so the returning borrower requests another. - missed_event: bool, closed: bool, pending_error: Option<(Arc, Error)>, pending_timeout: bool, @@ -64,30 +55,17 @@ struct Entry { published: bool, } -/// Unpublished objects removed together, then destroyed outside the device lock. -pub(super) struct UnusedRenderSpare { - _entry: Option, - _binding: Option, -} - pub(super) struct RenderState { config: Config, entries: KVec>, pools: pools::Pools, grow: grow::Grow, retired: KVec, - /// Graphs of released contexts the firmware may still walk (see `detach_context`): kept - /// until the firmware stops, when this state is dropped. - kept_graphs: KVec, next_pm_generation: u64, } impl RenderState { - pub(super) fn new( - clusters: u32, - deflake: crate::g17::fw::render::Deflake, - descriptor_flags: [u32; 2], - ) -> Result { + pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { if clusters == 0 { return Err(EINVAL); } @@ -98,14 +76,12 @@ impl RenderState { Ok(Self { config: Config { clusters, - deflake, descriptor_flags, }, entries, pools: pools::Pools::new()?, grow: grow::Grow::new()?, retired: KVec::with_capacity(128, GFP_KERNEL)?, - kept_graphs: KVec::new(), next_pm_generation: 1, }) } @@ -148,14 +124,6 @@ impl RenderState { .ok_or(EIO) } - /// Whether a completion event arrived while the pair in `slot` was - /// borrowed; clears the record. The returned pair is installed again, so - /// the next event pass observes it. - fn take_missed_event(&mut self, slot: u8) -> bool { - self.entry_mut(slot) - .is_ok_and(|entry| core::mem::take(&mut entry.missed_event)) - } - fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { let entry = self.entry_mut(reservation.slot)?; if entry.reservation.ids != reservation.ids @@ -189,7 +157,6 @@ impl RenderState { } entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); - entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet.clone()); entry.pair.take().ok_or(EIO) } @@ -209,7 +176,6 @@ impl RenderState { } entry.pair = pair.take(); entry.borrowed_dependencies = None; - entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -232,155 +198,11 @@ impl RenderState { } impl Registry { - /// Yield unused warming only when admission has run out of physical slots. - /// Previously published graphs keep their existing ownership and retirement - /// rules. The caller destroys returned backing after dropping the device lock. - pub(super) fn reclaim_unused_render_spare( - &mut self, - owner: Option, - require_ids: bool, - ) -> Result> { - // Prefer an empty reservation or an already parked channel. - for with_backing in [false, true] { - for slot in 1..RENDER_SLOTS { - let mut warm = false; - let expected = if let Some(entry) = self.render.entries[slot].as_ref() { - if entry.constructing - || entry.preparing.is_some() - || entry.growing - || entry.pending_error.is_some() - || entry.pending_timeout - { - continue; - } - // Compute can gain identities only from physical channels - // that never entered command publication. Render can reuse - // a retained render identity without changing its lifetime. - if owner.is_none() - && !entry - .reservation - .ids - .iter() - .all(|id| self.qids.can_cancel_unpublished(*id)) - { - continue; - } - if let Some(pair) = entry.pair.as_ref() { - if !with_backing { - continue; - } - if owner.is_some() && pair.reusable()? { - None - } else if pair.can_yield_unused()? { - warm = true; - Some(entry.reservation.owner) - } else { - continue; - } - } else { - if with_backing || entry.context.is_some() { - continue; - } - None - } - } else { - if with_backing || require_ids || owner.is_none() { - continue; - } - None - }; - if !self - .admission - .reassign_unused_spare(slot as u8, expected, owner) - { - continue; - } - if owner.is_none() { - let entry = self.render.entries[slot].take(); - if let Some(entry) = entry.as_ref() { - for id in entry.reservation.ids { - self.qids.cancel_unpublished(id)?; - } - if let Some(pair) = entry.pair.as_ref() { - self.buffers.unreserve_owner(pair.buffer_id())?; - } - self.note_build_room(); - } - return Ok(Some(UnusedRenderSpare { - _entry: entry, - _binding: None, - })); - } - let mut binding = None; - if warm { - let entry = self.render.entries[slot].as_mut().ok_or(EIO)?; - let mut pair = entry.pair.take().ok_or(EIO)?; - self.buffers.unreserve_owner(pair.buffer_id())?; - pair.release_owner(); - let (parked, retired) = KBox::into_inner(pair).park(); - entry.parked = Some(parked); - entry.context = None; - entry.pool = None; - binding = Some(retired); - } - return Ok(Some(UnusedRenderSpare { - _entry: None, - _binding: binding, - })); - } - } - Ok(None) - } - - /// The installed pair in `slot` belongs to `owner` and its context, is - /// neither borrowed nor settling a failure, and its pool has backing: the - /// state in which building the pair would do nothing. - pub(super) fn render_ready(&self, slot: u8, owner: u64, context: &Arc) -> Result { - let Ok(entry) = self.render.entry(slot) else { - return Ok(false); - }; - if entry.pending_error.is_some() - || entry.pending_timeout - || entry.constructing - || entry.preparing.is_some() - || entry.growing - || entry.parked.is_some() - { - return Ok(false); - } - let Some(pair) = entry.pair.as_ref() else { - return Ok(false); - }; - Ok(pair.matches(owner, context)? && !pair.pool().vacant()) - } - - fn has_render_ids(&self, owner: u64) -> bool { - self.render - .entries - .iter() - .enumerate() - .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) - } - - /// Tell admission whether another render pair can still be built. - pub(super) fn note_build_room(&self) { - self.admission.note_build_room(self.qids.render_pair_fits()); - } - - pub(super) fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { + fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); } - if self.render.entries[usize::from(slot)].is_some() { - return Ok(()); - } - let built = self.build_render_ids(slot, owner); - self.note_build_room(); - built - } - - fn build_render_ids(&mut self, slot: u8, owner: u64) -> Result { - { + if self.render.entries[usize::from(slot)].is_none() { let identity = self.identities.reserve(2)?.start; let tiling = self.qids.reserve(identity, DataMaster::Tiling, None)?; let fragment = match self.qids.reserve(identity + 1, DataMaster::Fragment, None) { @@ -406,9 +228,7 @@ impl Registry { pending_owner_closed: false, borrowed_active: false, borrowed_dependencies: None, - borrowed_switch: None, event_masks: [0; 2], - missed_event: false, closed: false, pending_error: None, pending_timeout: false, @@ -424,13 +244,6 @@ impl Registry { /// ensure_render_ids cancels a first ID if the second cannot be reserved; /// validation, identity overflow, and cancellation errors still propagate. fn prepare_render_ids(&mut self, slot: u8, owner: u64, optional: bool) -> Result { - // A full queue table refuses a sibling without scanning it on every preparation. - if optional - && self.render.entries.get(usize::from(slot)).is_some_and(Option::is_none) - && !self.qids.render_pair_fits() - { - return Ok(false); - } match self.ensure_render_ids(slot, owner) { Ok(()) => Ok(true), Err(error) if optional && error == ENOSPC => Ok(false), @@ -449,8 +262,12 @@ impl Registry { { return Err(EBUSY); } - // Construction keeps identities reserved. Publication pins them at the - // first firmware-visible command, so unused warming can still yield. + let ids = self.render.entry(slot)?.reservation.ids; + // Physical queue identities survive failed graph construction. Only + // failure to acquire the second identity can return the first one. + for id in ids { + self.qids.publish(id)?; + } let buffer = self.buffers.allocate_render()?; let entry = self.render.entry_mut(slot)?; entry.reservation.owner = owner; @@ -557,26 +374,32 @@ impl crate::g17::queue::render::Host for Host<'_> { core::sync::atomic::fence(Ordering::Acquire); Ok(stamps) } - fn qos_publish_pair( + fn qos_publish( &mut self, - owners: [crate::g17::qos::Owner; 2], + owner: crate::g17::qos::Owner, scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result<[crate::g17::qos::Publication; 2]> { - self.firmware.port.qos_publish_pair(owners, scheduler, policy) + ) -> Result { + use crate::g17::qos; + self.firmware.queues.accounting.publish( + &qos::View::new(self.firmware.init.qos()?)?, + owner, + scheduler, + policy, + qos::clock, + ) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { - self.firmware.port.qos_cancel(publication) + use crate::g17::qos; + self.firmware + .queues + .accounting + .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) } /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { - // Firmware builds without MMIO registration install the pair from its queue - // configuration alone. - if !self.firmware.variant.mmio_pair_registration { - return Ok(()); - } if !self.registration_powered { self.firmware.acquire_registration_power()?; self.registration_powered = true; @@ -596,9 +419,6 @@ impl crate::g17::queue::render::Host for Host<'_> { fn publish_qid(&mut self, id: kick::Id) -> Result { self.firmware.queues.qids.publish(id) } - fn next_publication(&mut self) -> Result { - Ok(self.firmware.port.next_publication()) - } fn publish_outer_pair( &mut self, priority: u8, @@ -638,9 +458,6 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); - crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { - self.firmware.bridge.regs.g17_core_power() - }); let result = self .firmware .primary @@ -660,20 +477,13 @@ impl crate::g17::queue::render::Host for Host<'_> { generation: u64, buffer_id: u8, ) -> Result { + use crate::g17::qos; let queues = &mut self.firmware.queues; - // Each final release is a control record the firmware processes before - // it returns to the work rings. While other pairs keep it busy, a - // registration left idle lingers instead, and the owner's next render - // reuses it. On an otherwise idle device the release is cheaper than - // reusing a registration that went idle, so it is sent at once. - // The pair retiring here is lent out of its entry meanwhile. - let linger = queues.render.entries.iter().flatten().any(|entry| { - entry.pair.as_ref().is_some_and(|pair| pair.published_in_flight()) - }); - let next = queues.render.pools.completed(pool_id, generation, linger)?; + let next = queues.render.pools.completed(pool_id, generation)?; queues.buffers.can_release(buffer_id)?; + let view = qos::View::new(self.firmware.init.qos()?)?; if let Some(owners) = owners { - self.firmware.port.qos_complete(&owners)?; + queues.accounting.complete(&view, &owners)?; } queues.render.pools.commit_completed(pool_id, next); queues.buffers.release(buffer_id) @@ -706,30 +516,6 @@ impl crate::g17::Firmware { result } - /// Event-worker turn: ends free-list lingers that elapsed or whose owner - /// closed or moved on, publishes their releases, and returns the delay to - /// the next expiry, which the worker turns into a timed wake. - pub(in crate::g17) fn expire_render_lingers(&mut self, now: u64) -> Result> { - let render = &mut self.queues.render; - let entries = &render.entries; - let (ended, next) = render.pools.expire_lingers(now, |owner| { - entries.iter().flatten().any(|entry| { - !entry.closed - && entry - .pool - .as_ref() - .is_some_and(|pool| pools::Pools::owner(pool) == *owner) - }) - }); - if ended { - match self.release_render_lists() { - Err(error) if error == EAGAIN => {} - result => result?, - } - } - Ok(next) - } - fn release_render_lists(&mut self) -> Result { if self.queues.render.grow.reply.is_some() { return Err(EAGAIN); @@ -767,57 +553,6 @@ impl crate::g17::Firmware { Ok(()) } - /// A final retirement pass before a render watchdog attributes its timeout, as compute runs - /// scan_compute() first: a head that finished but whose completion the event worker has not - /// processed yet must not defer another VM's timeout. Pairs lent out, or holding a deferred - /// timeout or error, stay with the worker, which observes and settles them in its order. - pub(in crate::g17::runtime) fn scan_renders(&mut self, deferred: &mut DeferredBatch) -> Result { - if self.recovery.pending() - || self - .primary - .state - .shared - .crashed - .load(core::sync::atomic::Ordering::Acquire) - { - return Ok(()); - } - for slot in 1..self.queues.render.entries.len() { - if !self.queues.render.entries[slot] - .as_ref() - .is_some_and(|entry| { - entry - .pair - .as_ref() - .is_some_and(|pair| pair.published_in_flight()) - && entry.preparing.is_none() - && !entry.growing - && !entry.pending_timeout - && entry.pending_error.is_none() - }) - { - continue; - } - let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; - if let Some(pair) = self.queues.render.entries[slot] - .as_mut() - .and_then(|entry| entry.pair.as_mut()) - { - let qids = pair.qids(); - let stamps = [ - self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), - self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), - ]; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - pair.note_stamp_wait(stamps, state, crate::g17::now_ns())?; - } - self.retire_render_slot(slot, None, deferred)?; - self.queues.render.collect_retired(slot)?; - self.settle_render_terminal(slot, deferred)?; - } - Ok(()) - } - pub(in crate::g17) fn settle_render_vm( &mut self, status: &Arc, @@ -869,22 +604,6 @@ impl Registry { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_spared()) } - /// Marks the interrupted group of the pair that owns `qid`. - pub(in crate::g17) fn note_render_interrupted( - &self, - qid: u8, - stamp: u32, - ) -> crate::g17::recovery::Interrupted { - self.render - .entries - .iter() - .flatten() - .filter_map(|entry| entry.pair.as_ref()) - .find(|pair| pair.qids().contains(&qid)) - .map_or(crate::g17::recovery::Interrupted::None, |pair| { - pair.note_interrupted(stamp) - }) - } pub(in crate::g17) fn visit_render_owners( &self, qid: u8, @@ -1044,19 +763,9 @@ impl Registry { } pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { for entry in self.render.entries.iter_mut().flatten() { - let mut hit = false; for id in entry.reservation.ids { let qid = usize::from(id.qid()); - let bit = masks[qid / 64] & (1u64 << (qid % 64)); - entry.event_masks[qid / 64] |= bit; - hit |= bit != 0; - } - // Event passes skip a pair while preparation, growth or an - // observation has borrowed it. Without a record, a completion - // that arrived meanwhile waits for an unrelated later event or the - // one-millisecond rescan. - if hit && (entry.preparing.is_some() || entry.growing) { - entry.missed_event = true; + entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); } } } @@ -1070,10 +779,10 @@ impl Registry { |pair| pair.published_in_flight(), ); active - && (crate::g17::queue::render::completion_noted( - entry.reservation.ids.map(kick::Id::qid), - entry.event_masks, - ) || entry.pair.as_ref().is_some_and(|pair| pair.stamp_waiting())) + && entry.reservation.ids.into_iter().all(|id| { + let qid = usize::from(id.qid()); + entry.event_masks[qid / 64] & (1u64 << (qid % 64)) != 0 + }) }) } pub(in crate::g17) fn defer_render_timeout( @@ -1107,105 +816,4 @@ impl Registry { entry.pending_timeout = true; true } - - /// The oldest published group of every render pair, for cross-VM timeout attribution - /// (crate::g17::timeout). - pub(in crate::g17) fn render_timeout_heads( - &self, - ) -> impl Iterator + '_ { - self.render - .entries - .iter() - .flatten() - .filter_map(|entry| entry.pair.as_ref()) - .filter_map(|pair| pair.timeout_head()) - } - - /// A render watchdog blocked behind another VM's older unfinished work gets the same - /// bounded grace as compute; the oldest unfinished work is never excused. When the GPU - /// stalls on one client, the others' renders then outlive the culprit's teardown instead - /// of all timing out together. - pub(in crate::g17) fn render_timeout_decision( - &self, - packet: &Arc, - renewals: u8, - ) -> Option { - let own = self - .render - .entries - .iter() - .flatten() - .filter_map(|entry| entry.pair.as_ref()) - .find(|pair| pair.contains_packet(packet)) - .and_then(|pair| pair.timeout_head()); - let computes = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter_map(|queue| queue.timeout_head()); - // Without a head of its own the timeout is its own. - own.map(|own| { - crate::g17::timeout::attribute( - own, - renewals, - Iterator::chain(self.render_timeout_heads(), computes), - ) - }) - } -} - -impl RenderState { - pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "retired_render_bindings used={} total={}", self.retired.len(), self.retired.capacity())?; - writeln!(f, "firmware_linked_render_graphs kept={} leaked={}", self.kept_graphs.len(), - teardown::LEAKED_GRAPHS.load(core::sync::atomic::Ordering::Relaxed))?; - self.pools.write_capacity(f)?; - for (slot, entry) in self.entries.iter().enumerate() { - if let Some(context) = entry.as_ref().and_then(|entry| entry.context.as_ref()) { - context.write_capacity(f, "render", slot)?; - } - } - Ok(()) - } -} - -impl Registry { - /// Give idle private backing to another ready client under capacity pressure. - /// Keep graph/QID/context ownership; the next use rearms the same pool after - /// its consumed release, exactly as eviction-driven reclaim does. - pub(super) fn reclaim_idle_render_pools( - &mut self, init: &crate::g17::initdata::InitData, deferred: &mut DeferredBatch, - ) -> Result { - let render = &mut self.render; - render.pools.observe(init)?; - for entry in render.entries.iter().flatten() { - if !deferred.render_backing_room() { break; } - let Some(pool) = entry.pool.as_ref() else { continue; }; - // None means this pool never acquired a firmware registration; - // otherwise require its exact final release to have been consumed. - if pool.vacant() || !render.pools.released(u32::from(pool.id()), pool.control_va()) - || self.admission.owner_active(entry.reservation.owner) { continue; } - // A sibling or replacement may be building off-lock before it has - // installed its pool reference. Its logical owner also excludes us. - let mut idle = true; - for other in render.entries.iter().flatten().filter(|other| { - other.reservation.owner == entry.reservation.owner - || other.pool.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, pool)) - }) { - if other.constructing || other.growing || other.preparing.is_some() - || other.pending_error.is_some() || other.pending_timeout { - idle = false; - break; - } - if let Some(pair) = other.pair.as_ref() { - if !pair.idle_for_context(pair.context())? { idle = false; break; } - } - } - if idle { - if let Some(backing) = pool.vacate() { deferred.defer_render_backing(backing); } - } - } - Ok(()) - } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index 277534328406e..b283fb8e7d55c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -14,8 +14,8 @@ use kernel::prelude::*; const RETRY_LIMIT: u16 = 1000; pub(super) struct Grow { - pub(super) requests: KVec, - pub(super) active: Option, + requests: KVec, + active: Option, pub(super) reply: Option<(TvbGrowRequest, bool, u16)>, release_attempts: u16, } @@ -40,7 +40,7 @@ impl RenderState { entry.published && u32::from(entry.reservation.buffer.0) == request.buffer_slot && entry.context.as_ref().is_some_and(|context| { - request.vm_slot == -1 || request.vm_slot as u32 == context.id() + request.vm_slot == -1 || request.vm_slot as u32 == u32::from(context.id()) }) }) }) @@ -228,8 +228,6 @@ impl crate::g17::Shared { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); - entry.borrowed_switch = - entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); entry.growing = true; Some(entry.pair.take().ok_or(EIO)?) } @@ -271,16 +269,13 @@ impl crate::g17::Shared { } entry.pair = pair.take(); entry.borrowed_dependencies = None; - entry.borrowed_switch = None; entry.growing = false; - // The grow work item requests an event pass when it finishes. - entry.missed_event = false; } if firmware.queues.render.grow.active != Some(request) { return Err(EIO); } firmware.queues.render.grow.active = None; - self.note_changed(); + self.changed.notify_all(); if !lease.is_current() || request.halt_count != firmware.recovery.generation() { continue; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 57543be54e835..31bcb688c8001 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -1,22 +1,21 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Event-driven render snapshots. A pair's completion memory is read and the -//! pair retired in one device-mutex hold, inside the preparation epoch. +//! Event-driven render snapshots outside the device mutex. The preparation +//! epoch and exact pair owner survive every read, cancellation and retry. use super::super::DeferredBatch; +use crate::g17::queue::render::Pair; use core::sync::atomic::Ordering; use kernel::{prelude::*, sync::Arc}; impl crate::g17::Firmware { - /// The first slot from `start` whose pair has news to observe: published work in flight, the - /// fragment queue's notification seen (`completion_noted`), and an observation the pair accepts. - fn next_render_observation(&mut self, start: usize) -> Result> { + fn take_render_observation(&mut self, start: usize) -> Result)>> { if self.recovery.pending() { return Ok(None); } let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; for slot in start..self.queues.render.entries.len() { - let Some(entry) = self.queues.render.entries[slot].as_ref() else { + let Some(entry) = self.queues.render.entries[slot].as_mut() else { continue; }; if entry.preparing.is_some() || entry.growing { @@ -27,8 +26,9 @@ impl crate::g17::Firmware { }; let qids = pair.qids(); if !pair.published_in_flight() - || (!crate::g17::queue::render::completion_noted(qids, entry.event_masks) - && !pair.stamp_waiting()) + || !qids + .into_iter() + .all(|qid| entry.event_masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) { continue; } @@ -37,22 +37,33 @@ impl crate::g17::Firmware { self.init.stamp(qids[1])?.load(Ordering::Relaxed), ]; core::sync::atomic::fence(Ordering::Acquire); - if pair.observation_packet(entry.event_masks, stamps, state)?.is_some() { - return Ok(Some(slot)); - } - // The stamps gate this pair out. If its front executed and was consumed anyway, the - // stamps may never move: after the bound, observe it and retire it on its record. - let pair = self.queues.render.entries[slot] - .as_mut() - .and_then(|entry| entry.pair.as_mut()) - .ok_or(EIO)?; - if pair.note_stamp_wait(stamps, state, crate::g17::now_ns())? { - return Ok(Some(slot)); - } + let Some(packet) = pair.observation_packet(entry.event_masks, stamps, state)? else { + continue; + }; + entry.borrowed_active = pair.published_in_flight(); + entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.preparing = Some(packet); + return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); } Ok(None) } + fn return_render_observation(&mut self, slot: usize, pair: &mut Option>) -> Result { + let returning = pair.as_ref().ok_or(EINVAL)?; + let entry = self.queues.render.entry_mut(slot as u8)?; + if entry.pair.is_some() + || entry.preparing.is_none() + || returning.slot() as usize != slot + || returning.qids() != entry.reservation.ids.map(crate::g17::kick::Id::qid) + { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.preparing = None; + Ok(()) + } + pub(super) fn settle_render_terminal( &mut self, slot: usize, @@ -109,36 +120,57 @@ impl crate::g17::Shared { Err(error) if error == EAGAIN || error == ENODEV => break, result => result?, }; - // Selecting, reading and retiring the front share one hold: a - // separate hold for the read would let publishers and the - // retirement contend twice per observed pair. - let mut state = self.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); + let loan = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if !epoch.is_current() || self.crashed.load(Ordering::Acquire) { + break; + } + firmware.take_render_observation(cursor)? }; - if !epoch.is_current() - || firmware.recovery.pending() - || self.crashed.load(Ordering::Acquire) - { - break; - } - let Some(slot) = firmware.next_render_observation(cursor)? else { + let Some((slot, pair)) = loan else { break; }; if slot != cursor { remaining = crate::hw::t8140::queues::RENDER_DEPTH; retries = 0; } - let observed = firmware.queues.render.entries[slot] - .as_ref() - .and_then(|entry| entry.pair.as_ref()) - .ok_or(EIO)? - .observe_front(); - // Publishers waiting for render capacity recheck once this hold ends. - self.note_changed(); - if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { - // Observation and the growth check run on every pass. - self.queue_event_records(); + let observed = pair.observe_front(); + let mut pair = Some(pair); + let mut state = self.state.lock(); + // Return installed ownership before examining read errors or epoch. + let restored = (*state) + .as_deref_mut() + .ok_or(ENODEV) + .and_then(|firmware| firmware.return_render_observation(slot, &mut pair)); + if let Err(error) = restored { + // A failed invariant must not drop firmware-owned storage. Join + // the other preparers only after releasing our own epoch, and + // keep this detached owner until both processors have stopped. + drop(state); + drop(epoch); + self.lose_device_quietly(); + self.stop_queues(); + if let Some(mut pair) = pair { + pair.stopped(&mut |event| { + event.finish(); + Ok(()) + })?; + } + return Err(error); + } + let firmware = (*state).as_deref_mut().ok_or(EIO)?; + self.changed.notify_all(); + if firmware.render_grow_pending() { + self.queue_events(); + } + if !epoch.is_current() + || firmware.recovery.pending() + || self.crashed.load(Ordering::Acquire) + { + break; } let result = (|| { let state = crate::g17::recovery::Memory::recovery_state(&firmware.init)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 1e3354d90dbfb..f7f48e853f225 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -6,7 +6,7 @@ use crate::g17::{ channel::Rings, - freelist::{Lease, Owner, RenderPool, LINGER_NS}, + freelist::{Lease, Owner, RenderPool}, fw::{ channels::{ControlRecord, FreeListCompletion}, initdata::PagePoolDescriptor, @@ -49,7 +49,7 @@ impl Pools { Ok(slot) } - pub(super) fn owner(pool: &RenderPool) -> Owner { + fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), epoch: pool.epoch(), @@ -154,12 +154,12 @@ impl Pools { } /// Preflight a paired retirement before QoS or PB reference accounting. - pub(super) fn completed(&self, id: u16, generation: u64, linger: bool) -> Result { + pub(super) fn completed(&self, id: u16, generation: u64) -> Result { let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; if next.generation() != generation { return Err(EIO); } - next.complete_render(linger)?; + next.complete_render()?; Ok(next) } @@ -169,32 +169,6 @@ impl Pools { self.leases[usize::from(id)] = Some(next); } - /// Ends the linger of every idle registration that `keep` (an open owner - /// still bound to the pool) does not retain or whose linger has elapsed. - /// Returns whether a release became pending, and the next expiry delay. - pub(super) fn expire_lingers( - &mut self, - now: u64, - keep: impl Fn(&Owner) -> bool, - ) -> (bool, Option) { - let mut ended = false; - let mut next: Option = None; - for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { - let Some(since) = lease.lingering() else { - continue; - }; - let age = now.saturating_sub(since); - if age >= LINGER_NS || !keep(&lease.owner()) { - lease.end_linger(); - ended = true; - } else { - let delay = LINGER_NS - age; - next = Some(next.map_or(delay, |next| next.min(delay))); - } - } - (ended, next) - } - pub(super) fn pending(&self) -> bool { self.leases .iter() @@ -212,7 +186,7 @@ impl Pools { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); - for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { + for lease in self.leases.iter_mut().flatten() { lease.observe_releases(consumer, producer)?; } Ok(()) @@ -225,7 +199,7 @@ impl Pools { init: &InitData, mut announce: impl FnMut() -> Result, ) -> Result { - for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { + for lease in self.leases.iter_mut().flatten() { let Some(release) = lease.pending_release() else { continue; }; @@ -265,11 +239,3 @@ fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { table.dword(offset + 24)?, ]) } - -impl Pools { - pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "render_pool_leases used={} total={} retained={}", - self.leases.iter().flatten().filter(|lease| !lease.released()).count(), - POOLS - FIRST_RENDER_POOL, self.leases.iter().flatten().count()) - } -} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 62ca7895a86f8..06bea3c84fa7a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -16,7 +16,6 @@ use kernel::{prelude::*, sync::Arc}; struct BuildResources { uat: Arc, objects: Arc, - rings: Arc, ids: Arc, global: crate::mmu::Vm, metrics: Arc, @@ -29,14 +28,13 @@ impl BuildResources { Ok(Self { uat: firmware.uat.clone(), objects: firmware.pool.clone(), - rings: firmware.render_rings.clone(), ids: firmware.render_ids.clone(), global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), metrics_ids: firmware.queues.metrics.clone(), metrics_bases: crate::g17::fw::buffer::MetricsAddresses { firmware: firmware.init.metrics_fw_va()?, - client: firmware.init.pm_metrics_va(), + client: crate::hw::t8140::CONFIG.pm_metrics.va, }, config: firmware.queues.render.config, }) @@ -46,10 +44,6 @@ impl BuildResources { } } -#[expect( - clippy::large_enum_variant, - reason = "Keep preparation allocation-free; boxing adds a fallible allocation to submission." -)] enum Build { Fresh(Reservation, u64, MetricsLease), Parked { @@ -78,14 +72,8 @@ struct BuildContext<'a> { } impl super::super::Backend { - fn build_render_pair( - &self, - slot: u8, - first: &Arc, - optional: bool, - sleep: bool, - ) -> Result { - let lease = self.enter_preparation(sleep)?; + fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { + let lease = self.shared.preparations.enter(false)?; let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); @@ -145,13 +133,6 @@ impl super::super::Backend { if !pair.reusable()? { return Err(EBUSY); } - // The release of the pair's scheduler is armed or still being - // consumed: the firmware may process it against these queues. - // The teardown parks the pair once the release is witnessed; - // the new owner then installs it with fresh queue records. - if firmware.queues.teardown.releasing(pair.context()) { - return Err(EBUSY); - } let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); if foreign { firmware @@ -183,8 +164,6 @@ impl super::super::Backend { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); - entry.borrowed_switch = - entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); Build::Rebind { pair: entry.pair.take().ok_or(EIO)?, buffer, @@ -259,11 +238,10 @@ impl super::super::Backend { let built = (|| -> Result> { let free_list = self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + .render_pool(&alloc, &resources.ids, &resources.global)?; Pair::new( - alloc, + &alloc, &resources.objects, - &resources.rings, reservation.ids, slot, self.owner, @@ -274,9 +252,7 @@ impl super::super::Backend { pm_generation, first, resources.config.clusters, - resources.config.deflake, resources.config.descriptor_flags, - self.shared.variant, ) })(); let (mut built, error) = match built { @@ -301,7 +277,7 @@ impl super::super::Backend { built .as_mut() .ok_or(EIO)? - .finish_install(alloc, firmware.init.pb_descriptor_table()?)?; + .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; firmware.queues.install_render(reservation, &mut built) })(); // Fresh backing disappears before its temporary PB reference returns. @@ -340,9 +316,9 @@ impl super::super::Backend { let built = (|| -> Result<_> { let free_list = self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + .render_pool(&alloc, &resources.ids, &resources.global)?; let prepared = parked.prepare( - alloc, + &alloc, &resources.objects, self.owner, self.context.clone(), @@ -384,7 +360,7 @@ impl super::super::Backend { firmware.queues.buffers.can_release(buffer.0)?; prepared.as_mut().ok_or(EIO)?.finish_install( &parked, - alloc, + &alloc, firmware.init.pb_descriptor_table()?, )?; let pair = parked.commit(&mut prepared, storage)?; @@ -421,12 +397,9 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; - if firmware.queues.render.take_missed_event(slot) { - self.shared.queue_events(); - } firmware.render_grow_ready() }; - self.shared.note_changed(); + self.shared.changed.notify_all(); if grow_ready { self.shared.queue_grow(); } @@ -456,9 +429,9 @@ impl super::super::Backend { let prepared = (|| { let free_list = self.context - .render_pool(alloc, &resources.ids, &resources.global)?; + .render_pool(&alloc, &resources.ids, &resources.global)?; pair.prepare_bind( - alloc, + &alloc, &resources.objects, self.owner, self.context.clone(), @@ -490,14 +463,10 @@ impl super::super::Backend { if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { return Err(ECANCELED); } - // That owner may have closed while this binding was prepared. - if firmware.queues.teardown.releasing(pair.context()) { - return Err(EBUSY); - } if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RetiredRenderBindings)); + return Err(ENOSPC); } firmware.queues.teardown.displace( pair.context(), @@ -506,7 +475,7 @@ impl super::super::Backend { )?; pair.bind_owner( prepared.as_mut().ok_or(EIO)?, - alloc, + &alloc, firmware.init.pb_descriptor_table()?, )?; if let Some((id, _)) = buffer { @@ -530,10 +499,6 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; - entry.borrowed_switch = None; - if firmware.queues.render.take_missed_event(slot) { - self.shared.queue_events(); - } (result, firmware.render_grow_ready()) }; drop(prepared); @@ -548,7 +513,7 @@ impl super::super::Backend { .release(id)?; } } - self.shared.note_changed(); + self.shared.changed.notify_all(); if grow_ready { self.shared.queue_grow(); } @@ -591,12 +556,8 @@ impl super::super::Backend { firmware.queues.render.pools.warm(&firmware.init, &pool) } - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { - self.build_render_pair(slot, packet, false, sleep)?; - // An owner has exactly one pair (kick.rs asserts it): no sibling to warm. - if crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1 { - return Ok(()); - } + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { + self.build_render_pair(slot, packet, false)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -607,12 +568,10 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { - // The host lease excludes capacity reclamation until this off-lock - // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling.slot(), packet, true, sleep) { - Err(error) if error == EBUSY || error == EAGAIN || error == super::submit::FALLBACK => {} + match self.build_render_pair(sibling, packet, true) { + Err(error) if error == EBUSY || error == EAGAIN => {} result => result?, } } @@ -624,9 +583,8 @@ impl super::super::Backend { slot: u8, packet: &Arc, deferred: &mut super::super::DeferredBatch, - sleep: bool, ) -> Result { - let lease = self.enter_preparation(sleep)?; + let lease = self.shared.preparations.enter(false)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); @@ -702,9 +660,6 @@ impl super::super::Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.queues.render.return_pair(slot, &mut returned)?; - if firmware.queues.render.take_missed_event(slot) { - self.shared.queue_events(); - } if firmware.queues.render.entry(slot)?.closed { firmware .queues @@ -715,7 +670,7 @@ impl super::super::Backend { .ok_or(EIO)? .release_owner(); } - self.shared.note_changed(); + self.shared.changed.notify_all(); let published = (|| { let entry = firmware.queues.render.entry(slot)?; if entry.pending_error.is_some() || entry.pending_timeout { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index 499e3858fcca1..ce9771778acd4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -2,8 +2,8 @@ //! Republication of render groups that firmware retired without executing. //! The pair proves the abandonment (retired, unlinked, stamps covering it, no -//! recovery, fragment end never written) and parks the packets in order; this -//! pass prepares and publishes the same commands again under the device mutex. +//! recovery, fragment end never written) and parks the packet; this pass +//! prepares and publishes the same command again under the device mutex. use super::super::DeferredBatch; use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; @@ -27,91 +27,79 @@ impl Firmware { return Ok(()); } for slot in 0..self.queues.render.entries.len() { - // Publish the parked groups front first; a pause keeps the rest parked. - while self.replay_render_front(slot, deferred)? {} - } - Ok(()) - } - - /// Publishes the oldest parked group of `slot`; returns whether it was published. - fn replay_render_front(&mut self, slot: usize, deferred: &mut DeferredBatch) -> Result { - let Some((packet, owner, qids, closed, busy, stalled)) = - self.queues.render.entries[slot].as_ref().and_then(|entry| { - let pair = entry.pair.as_ref()?; - let packet = pair.replay_front()?; - Some(( - packet, - entry.reservation.owner, - pair.qids(), - entry.closed, - entry.preparing.is_some() - || entry.growing - || entry.pending_timeout - || entry.pending_error.is_some(), - pair.replay_stalled(crate::g17::now_ns()), - )) - }) - else { - return Ok(false); - }; - if stalled { - return Err(ETIMEDOUT); - } - if busy { - return Ok(false); - } - if closed || packet.cancel_requested() || !packet.context.is_current() { - self.with_render(slot, |pair, _| { - pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) - })?; - return Ok(false); - } - if packet.completion.status().get() != 0 { - self.with_render(slot, |pair, _| { - pair.fail_replay(EIO, &mut |event| deferred.push(event)) - })?; - return Ok(false); - } - let addresses = RenderAddresses { - stamps: [ - self.init.stamp_addresses(qids[0])?, - self.init.stamp_addresses(qids[1])?, - ], - }; - let dev = self.primary.state.shared.drm()?; - let uat = self.uat.clone(); - let prepared = self.with_render(slot, |pair, _| { - if !pair.room() { - return Err(EBUSY); - } - let alloc = Allocator { - dev: &dev, - uat: &uat, + let Some((packet, owner, qids, closed, busy)) = + self.queues.render.entries[slot].as_ref().and_then(|entry| { + let pair = entry.pair.as_ref()?; + let packet = pair.replay_front()?; + Some(( + packet, + entry.reservation.owner, + pair.qids(), + entry.closed, + entry.preparing.is_some() + || entry.growing + || entry.pending_timeout + || entry.pending_error.is_some(), + )) + }) + else { + continue; }; - pair.prepare(&alloc, &packet, addresses) - }); - match prepared { - Ok(()) => {} - Err(error) if error == EBUSY || error == EAGAIN => return Ok(false), - Err(error) => { + if busy { + continue; + } + if closed || packet.cancel_requested() || !packet.context.is_current() { self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) + pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) })?; - return Ok(false); - } - } - match self.publish_render(owner, slot as u8, &packet, deferred) { - Ok(()) => { - self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; - Ok(true) + continue; } - Err(error) if error == EBUSY || error == EAGAIN => Ok(false), - Err(error) => { + if packet.completion.status().get() != 0 { self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) + pair.fail_replay(EIO, &mut |event| deferred.push(event)) })?; - Ok(false) + continue; + } + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let dev = self.primary.state.shared.drm()?; + let uat = self.uat.clone(); + let prepared = self.with_render(slot, |pair, _| { + if !pair.room() { + return Err(EBUSY); + } + let alloc = Allocator { + dev: &dev, + uat: &uat, + }; + pair.prepare(&alloc, &packet, addresses) + }); + match prepared { + Ok(()) => {} + Err(error) if error == EBUSY || error == EAGAIN => continue, + Err(error) => { + self.with_render(slot, |pair, _| { + pair.fail_replay(error, &mut |event| deferred.push(event)) + })?; + continue; + } + } + match self.publish_render(owner, slot as u8, &packet, deferred) { + Ok(()) => { + self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; + } + Err(error) if error == EBUSY || error == EAGAIN => continue, + Err(error) => { + self.with_render(slot, |pair, _| { + pair.fail_replay(error, &mut |event| deferred.push(event)) + })?; + } } } + Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 71f0bfc052696..2e88848b93216 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -11,11 +11,7 @@ use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; impl Backend { pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - // Without a collector the watchdog expires again and retries: memory - // pressure must not stop the runtime for every client. - let Ok(mut deferred) = DeferredBatch::worker() else { - return Ok(()); - }; + let mut deferred = DeferredBatch::worker()?; let result = { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -37,7 +33,7 @@ impl Backend { kernel::bindings::dma_fence_wait_timeout( fence.raw(), false, - kernel::time::msecs_to_jiffies(2000) as isize, + kernel::time::msecs_to_jiffies(2000) as _, ) }; if completed > 0 { @@ -79,20 +75,6 @@ impl crate::g17::Firmware { } let generation = self.recovery.generation(); let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; - if error == ETIMEDOUT { - let qids = pair.qids(); - let stamps = [ - self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), - self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), - ]; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - if let Some(pair) = self.queues.render.entries[slot] - .as_mut() - .and_then(|entry| entry.pair.as_mut()) - { - pair.expire_stamp_wait(packet, stamps, state)?; - } - } let progressed = self.with_render(slot, |pair, host| { pair.retire(None, generation, state, host, &mut |event| { deferred.push(event) @@ -139,14 +121,6 @@ impl crate::g17::Firmware { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_packet()); if let Some(packet) = packet { - // Unclassified work counts as innocent, and a blamed group must never be replayed. - // Work an earlier recovery spared is collateral here (classify() blames before its - // collateral check) and stays spared, so its failure stays non-sticky; the blame - // itself is recorded apart from the class, and refuses any replay of it. - packet.completion.note_blamed(); - if !packet.completion.spared() { - packet.completion.classify(crate::g17::recovery::Class::Guilty); - } self.fail_render_packet(&packet, error, deferred)?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 970035547dfee..13381b62da86c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -8,31 +8,13 @@ use crate::g17::{ queue::admission::{RenderLease, Request}, }; use core::sync::atomic::Ordering; -use kernel::{prelude::*, sync::Arc}; - -/// Internal result of a publication attempted from the submitting ioctl that -/// met contention before anything became visible: the scheduler publishes the -/// command instead. Never returned to userspace. -pub(in crate::g17::runtime) const FALLBACK: Error = EINPROGRESS; +use kernel::{ + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta}, +}; impl Backend { - /// The submitting ioctl holds the logical queue's mutex and never sleeps - /// for contention or a closed preparation gate: it returns FALLBACK, and - /// the scheduler's worker publishes the command and owns the waits. - pub(in crate::g17::runtime) fn enter_preparation(&self, sleep: bool) -> Result { - if sleep { - return self.shared.preparations.enter(false); - } - // A closed gate (recovery) is the only case enter() would sleep for. - self.shared.preparations.try_enter().map_err(|error| { - if error == EAGAIN { - FALLBACK - } else { - error - } - }) - } - pub(in crate::g17) fn render_packet( &self, packet: &Arc, @@ -41,312 +23,6 @@ impl Backend { self.render_with_lease(packet, None, deferred) } - /// Reserve identities before attaching a lease to the packet. An empty - /// logical slot must not hide reusable physical channels when the shared - /// queue-ID namespace is full. - /// Also reports whether the owner's installed pair is ready to take the - /// command, so that the publication skips graph construction. - fn acquire_render_with_ids( - &self, - packet: &Arc, - mut next: Option, - sleep: bool, - ) -> Result<(RenderLease, bool)> { - let pool = &self.admission; - let started = super::super::now_ns(); - let mut deadline_ns = - started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); - let recovery_deadline_ns = started.saturating_add(recovery_wait_ns()); - // A capacity refusal samples the render domain and retries once; only - // a refusal after that sample waits, so no release can be missed. - let mut observed = None; - let mut capacity = WaitClock::default(); - loop { - self.admission_open()?; - if super::super::now_ns() >= deadline_ns { - if deadline_ns < recovery_deadline_ns && self.waiting_for_recovery()? { - deadline_ns = recovery_deadline_ns; - } else { - return Err(EBUSY); - } - } - if packet.cancel_requested() { - return Err(ECANCELED); - } - if packet.completion.status().get() != 0 { - return Err(EIO); - } - let lease = if let Some(lease) = next.take() { - lease - } else { - // Admission sleeps for a pick; the ioctl leaves that to the scheduler. - if !sleep { - return Err(FALLBACK); - } - match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { - // Atomics only: the callback runs under the pool mutex. - if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { - return Err(ECANCELED); - } - if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { - return Err(EIO); - } - Ok(()) - }) { - Ok(lease) => lease, - Err(error) if error == ENOSPC => { - let reclaimed = { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .reclaim_unused_render_spare(Some(self.owner), false)? - }; - if reclaimed.is_none() { - self.wait_render_capacity(&mut observed, &mut capacity, sleep)?; - } - drop(reclaimed); - continue; - } - // The deadline check above decides whether a recovery extends the wait. - Err(error) if error == EBUSY => continue, - Err(error) => return Err(error), - } - }; - let slot = lease.slot(); - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - match firmware.queues.ensure_render_ids(slot, self.owner) { - Ok(()) => { - let ready = firmware.queues.render_ready(slot, self.owner, &self.context)?; - return Ok((lease, ready)); - } - Err(error) if error == ENOSPC => { - let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; - if reclaimed.is_some() { - drop(state); - drop(reclaimed); - next = Some(lease); - continue; - } - // ensure_render_ids leaves no partial entry on refusal. - if firmware.queues.render.entries[usize::from(slot)].is_some() { - return Err(error); - } - if !lease.cancel_empty(self.owner) { - continue; - } - let reclaimed = firmware - .queues - .reclaim_unused_render_spare(Some(self.owner), true)?; - let ready = firmware.queues.has_render_ids(self.owner); - drop(state); - // A full queue table makes admission reuse a free killed pair instead. - if reclaimed.is_none() && !ready && !pool.killed_pair_free() { - self.wait_render_capacity(&mut observed, &mut capacity, sleep)?; - } - drop(reclaimed); - } - Err(error) => return Err(error), - } - } - } - - /// Scheduler admission of a render packet. A work state of its context or - /// a pair admission it lacks becomes a fence that the return providing it - /// signals, so run_job finds both ready instead of waiting for them. - /// Errors, and anything else publication may wait for, are left to it. - pub(in crate::g17::runtime) fn admit_render(&self, packet: &Arc) -> Option { - if packet.early_result().is_some() - || packet.cancel_requested() - || packet.completion.status().get() != 0 - || self.admission_open().is_err() - { - return None; - } - // Each check runs once more after sampling its domain, so the common - // case takes no domain lock and no return between check and wait is lost. - let work = packet.completion.work_state().ok()?; - loop { - match work.acquire(crate::g17::job::Engine::Render) { - Err(error) if error == EBUSY => { - let states = work.returns().observe(); - match work.acquire(crate::g17::job::Engine::Render) { - Err(error) if error == EBUSY => match work.returns().wait(states) { - Ok(Some(fence)) => return Some(fence), - Ok(None) => continue, - Err(_) => return None, - }, - Err(_) => return None, - Ok(_) => {} - } - } - Err(_) => return None, - Ok(_) => {} - } - let request = Request::of(packet); - if self.admission.admits_render(self.owner, request) { - return None; - } - let observed = self.shared.renders.observe(); - if self.admission.admits_render(self.owner, request) { - return None; - } - self.shared.queue_events(); - match self.shared.renders.wait(observed) { - Ok(Some(fence)) => return Some(fence), - Ok(None) => continue, - Err(_) => return None, - } - } - } - - /// Conditions which end a publication that has not become visible yet. - fn publication_open(&self, packet: &Packet) -> Result { - self.admission_open()?; - if packet.cancel_requested() { - return Err(ECANCELED); - } - if packet.completion.status().get() != 0 { - return Err(EIO); - } - Ok(()) - } - - /// Blocks on `domain` until a notification after `observed`, or the - /// deadline, which only bounds the wait. Slices of at most 10 ms let the - /// publisher observe cancellation, which has no notification here. - fn wait_domain( - &self, - domain: &crate::g17::availability::Availability, - observed: u64, - deadline_ns: u64, - packet: &Packet, - ) -> Result { - self.publication_open(packet)?; - let remaining = deadline_ns.saturating_sub(super::super::now_ns()); - if remaining == 0 { - return Err(ETIMEDOUT); - } - domain.block(observed, remaining.div_ceil(1_000_000).min(10) as u32, false) - } - - /// A recovery is pending, or this owner's pair is spared and draining after one: an - /// innocent publication waits up to RECOVERY_WAIT_MS for it instead of failing. - fn waiting_for_recovery(&self) -> Result { - let state = self.shared.state.lock(); - let firmware = (*state).as_deref().ok_or(ENODEV)?; - Ok(firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 - || firmware.queues.admission.spared_quarantined(self.owner)) - } - - /// The deadline of the wait `clock` measures (WaitClock::deadline). - fn wait_deadline(&self, clock: &mut WaitClock, bound_ns: u64) -> Result { - let recovering = self.waiting_for_recovery()?; - Ok(clock.deadline(super::super::now_ns(), recovering, bound_ns)) - } - - /// A pair is borrowed, still settling, waiting for firmware room or for the - /// release of the scheduler it last ran for. Every pair return, retirement - /// and teardown step signals the render domain, and each event pass does - /// while a publisher waits; the deadline is only the failure bound. - fn wait_render_busy( - &self, - clock: &mut WaitClock, - observed: &mut Option, - sleep: bool, - packet: &Packet, - ) -> Result { - if !sleep { - return Err(FALLBACK); - } - let deadline = self.wait_deadline(clock, super::super::WORK_RING_WAIT_NS)?; - // Sample, then retry once: only a refusal after the sample waits. - let Some(sample) = observed.take() else { - *observed = Some(self.shared.renders.observe()); - return Ok(()); - }; - self.shared.queue_events(); - self.wait_domain(&self.shared.renders, sample, deadline, packet) - } - - /// Render capacity is short: return what idle owners hold, then block until - /// a pair, lease or owner returns. No packet is firmware-visible here, and - /// neither the device mutex nor a preparation lease is held while waiting. - fn wait_render_capacity( - &self, - observed: &mut Option, - clock: &mut WaitClock, - sleep: bool, - ) -> Result { - if !sleep { - return Err(FALLBACK); - } - if self.reclaim_render_capacity()? { - *observed = None; - return Ok(()); - } - let Some(sample) = observed.take() else { - *observed = Some(self.shared.renders.observe()); - return Ok(()); - }; - self.shared.queue_events(); - self.admission_open()?; - let remaining = self - .wait_deadline(clock, super::super::WORK_RING_WAIT_NS)? - .saturating_sub(super::super::now_ns()); - if remaining == 0 { - return Err(EBUSY); - } - self.shared - .renders - .block(sample, remaining.div_ceil(1_000_000).min(10) as u32, false) - } - - /// Gives idle render backing back under capacity pressure. Returns whether - /// anything was returned, in which case the caller retries at once. - fn reclaim_render_capacity(&self) -> Result { - let mut deferred = DeferredBatch::publication()?; - let result: Result = (|| { - let lease = self.shared.preparations.try_enter()?; - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { - return Err(EAGAIN); - } - if firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 - { - return Ok(false); - } - firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; - firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; - let mut published = 0; - if !firmware.render_control_backpressured() { - (published, _) = firmware.queues.vacate_idle_pools( - &firmware.init, - 0, - super::super::now_ns(), - 0, - )?; - if published != 0 { - firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; - } - } - Ok(published != 0 - || !deferred.render.is_empty() - || !deferred.vacated.is_empty() - || !deferred.bindings.is_empty()) - })(); - deferred.finish(); - match result { - Err(error) if error == EAGAIN => Ok(false), - result => result, - } - } - fn render_with_lease( &self, packet: &Arc, @@ -354,7 +30,6 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; - self.admission_open()?; if packet.cancel_requested() { return Err(ECANCELED); } @@ -362,48 +37,22 @@ impl Backend { return Err(EIO); } let interruptible = early.is_none(); - // A publication from the submitting ioctl (an early lease) never - // sleeps for contention; it falls back to the scheduler instead. - let sleep = early.is_none(); - let mut early = early; - // The pair this owner last published to directly: a lease on it goes - // straight to the direct path, whose lend checks everything the - // identity check would. - let known = self.direct_slot.load(Ordering::Relaxed); - let lease = match early.take() { - Some(lease) => Some(lease), - None if known != 0 => self - .admission - .try_acquire_render(self.owner, Request::of(packet)) - .ok() - .flatten(), - None => None, - }; - let early = match lease { - Some(lease) if u32::from(lease.slot()) == known => { - let slot = lease.slot(); - packet.completion.attach_render_lease(lease)?; - if let Some(result) = self.render_direct(packet, slot, deferred) { - return result; - } - packet.completion.detach_render_lease() + let admission = match early { + Some(lease) => lease, + None => { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + pool.acquire_render(self.owner, Request::of(packet))? } - lease => lease, }; - let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; - if ready { - if let Some(result) = self.render_direct(packet, slot, deferred) { - return result; - } - } - let mut busy = WaitClock::default(); - let mut capacity = WaitClock::default(); - let mut state_clock = WaitClock::default(); - let mut recovery_since = None; - let mut seen = None; - let mut states = None; + let mut state_polls = 0; + let mut state_delay = 20i64; + let mut takeover_polls = 0; + let mut recovery_polls = 0; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -416,12 +65,8 @@ impl Backend { return Err(EFAULT); } let work_state = { - let preparation = if sleep { - self.shared.preparations.enter(interruptible)? - } else { - self.enter_preparation(false)? - }; - let result = packet.completion.work_state().and_then(|state| state.acquire(crate::g17::job::Engine::Render)); + let preparation = self.shared.preparations.enter(interruptible)?; + let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result } else { @@ -431,77 +76,51 @@ impl Backend { match work_state { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if !sleep { - return Err(FALLBACK); - } - // Every node of this context belongs to unretired work: - // sample, retry once, then wait for the next returned one. - let work = packet.completion.work_state()?; - match states.take() { - None => states = Some(work.returns().observe()), - // Work states free as earlier, possibly spared, work retires. - Some(sample) => { - let deadline = self - .wait_deadline(&mut state_clock, super::super::WORK_RING_WAIT_NS)?; - self.wait_domain(work.returns(), sample, deadline, packet)? - } + if state_polls == 1000 { + return Err(ETIMEDOUT); } + state_polls += 1; + fsleep(Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); continue; } result => { result?; } } - // A pair found ready under the identity check needs no build; any - // retry revalidates through ensure_render. - let ensured = if core::mem::take(&mut ready) { - Ok(()) - } else { - self.ensure_render(packet, slot, sleep) - }; - match ensured { - Err(error) if error == ENOSPC => { - self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; - continue; - } + match self.ensure_render(packet, slot) { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; + if takeover_polls == 5000 { + return Err(ETIMEDOUT); + } + takeover_polls += 1; + fsleep(Delta::from_micros(200)); continue; } result => result?, } - match self.prepare_render(slot, packet, deferred, sleep) { - Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; - continue; - } + match self.prepare_render(slot, packet, deferred) { Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { - let (recovering, gate) = { + let recovering = { let state = self.shared.state.lock(); let firmware = (*state).as_deref().ok_or(ENODEV)?; - ( - firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0, - self.shared.preparations.snapshot(), - ) + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 }; if recovering { - if !sleep { - return Err(FALLBACK); + if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { + return Err(ETIMEDOUT); } - // Recovery closes the preparation gate before touching - // any queue; the next pass blocks in enter() until it - // reopens. The worker raises the closure. - let since = *recovery_since.get_or_insert(super::super::now_ns()); + recovery_polls += 1; self.shared.queue_events(); - self.shared - .preparations - .wait_closure(gate, since.saturating_add(recovery_wait_ns()))?; + fsleep(Delta::from_millis(1)); } else if error == EBUSY { - self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; - } else if !sleep { - return Err(FALLBACK); + if takeover_polls == 5000 { + return Err(ETIMEDOUT); + } + takeover_polls += 1; + fsleep(Delta::from_micros(200)); } } result => return result, @@ -513,9 +132,19 @@ impl Backend { if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { return false; } - let admission = match self.admission.try_acquire_render(self.owner, Request::of(packet)) { - Ok(Some(lease)) => lease, - _ => return false, + let admission = { + let state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref() else { + return false; + }; + match firmware + .queues + .admission + .try_acquire_render(self.owner, Request::of(packet)) + { + Ok(Some(lease)) => lease, + _ => return false, + } }; if !packet.begin_publication() { packet.completion.fail_unpublished(ECANCELED); @@ -534,16 +163,6 @@ impl Backend { } }; let result = self.render_with_lease(packet, Some(admission), &mut deferred); - if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { - // Contention before anything became visible: the ioctl does not - // sleep. Return the lease and the claim; the scheduler publishes - // the command, and later ones wait behind it in the backlog. - let lease = packet.completion.detach_render_lease(); - deferred.finish(); - drop(lease); - packet.abandon_publication(); - return false; - } let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -667,37 +286,16 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } - let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; - // Every render publication, with or without the device mutex, holds the - // port's render lock: the render work rings have one host producer at a - // time and attachment overlaps are ordered by publication order. - let port = self.port.clone(); - let _render = port.render_lock(); // Both outer destinations must have room before pool accounting or - // either inner queue becomes visible. + // either inner queue becomes visible. All host producers share this lock. let priority = packet.context.policy().priority(); if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } - if let Some(footprint) = packet.footprint.as_ref() { - dependencies.hazard_count = self.queues.admission.publication_barriers( - owner, - slot, - footprint, - &mut dependencies.hazards, - )?; - } + let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; - // An owner change taken from a live owner which closed meanwhile: its - // first command waits until that scheduler's release has been consumed. - if pair - .owner_switch_from(&packet.context) - .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) - { - return Err(EBUSY); - } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; @@ -737,57 +335,3 @@ impl crate::g17::Firmware { result } } - -fn recovery_wait_ns() -> u64 { - u64::from(crate::g17::timeout::RECOVERY_WAIT_MS) * 1_000_000 -} - -/// One bounded wait (Backend::wait_deadline). -#[derive(Default)] -struct WaitClock { - first: Option, - calm: u64, -} - -impl WaitClock { - /// `bound_ns` after the later of the first call and the last call that saw a recovery - /// pending or this owner's spared pair draining (so the usual bound applies afresh once a - /// recovery ends), and at most RECOVERY_WAIT_MS after the first call. - fn deadline(&mut self, now: u64, recovering: bool, bound_ns: u64) -> u64 { - let first = *self.first.get_or_insert(now); - self.calm = if recovering { now } else { self.calm.max(first) }; - first - .saturating_add(recovery_wait_ns()) - .min(self.calm.saturating_add(bound_ns)) - } -} - -#[kunit_tests(asahi_g17_wait_clock)] -mod tests { - use super::*; - - const BOUND: u64 = 2_000_000_000; - - #[test] - fn a_calm_wait_keeps_its_first_deadline() { - let mut clock = WaitClock::default(); - assert_eq!(clock.deadline(100, false, BOUND), 100 + BOUND); - assert_eq!(clock.deadline(500, false, BOUND), 100 + BOUND); - } - - #[test] - fn the_bound_starts_afresh_once_a_recovery_ends() { - let mut clock = WaitClock::default(); - clock.deadline(100, false, BOUND); - assert_eq!(clock.deadline(1_000, true, BOUND), 1_000 + BOUND); - assert_eq!(clock.deadline(2_000, false, BOUND), 1_000 + BOUND); - } - - #[test] - fn a_recovery_cannot_extend_past_the_recovery_wait() { - let mut clock = WaitClock::default(); - clock.deadline(100, true, BOUND); - let limit = 100 + recovery_wait_ns(); - assert_eq!(clock.deadline(limit - 1, true, BOUND), limit); - } -} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index fffa6d28dd289..759c0c2e43dc0 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -17,15 +17,6 @@ impl RenderState { } pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { - // A pair that changed owner keeps running this scheduler's queues until - // the new owner's first command leaves it, whichever context owns it now. - let scheduler = Some(context.scheduler_va()); - if self.entries.iter().flatten().any(|entry| match entry.pair.as_ref() { - Some(pair) => pair.switch_in_flight() == scheduler, - None => (entry.preparing.is_some() || entry.growing) && entry.borrowed_switch == scheduler, - }) { - return Ok(false); - } for entry in self.entries.iter().flatten() { if !entry .context @@ -38,7 +29,7 @@ impl RenderState { return Ok(false); } if let Some(pair) = entry.pair.as_ref() { - if !pair.idle_for_context(context)? && !pair.executed_for_kill(context) { + if !pair.idle_for_context(context)? { return Ok(false); } } @@ -46,14 +37,6 @@ impl RenderState { Ok(true) } - /// Some render pair runs another context's commands. - pub(in crate::g17::runtime) fn foreign_in_flight(&self, context: &Arc) -> bool { - self.entries.iter().flatten().any(|entry| { - !entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) - && entry.pair.as_ref().is_some_and(|pair| pair.in_flight()) - }) - } - pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { context.render_pools_released(|id, control| self.pools.released(id, control)) && self @@ -123,7 +106,6 @@ impl RenderState { if !out.has_room(count, 1) { return Err(EBUSY); } - let linked = context.firmware_may_link(); for entry in self.entries.iter_mut().flatten() { if !entry .context @@ -138,17 +120,8 @@ impl RenderState { buffers.unreserve_owner(pair.buffer_id())?; let target = out.binding_slot()?; let pair = entry.pair.take().ok_or(EIO)?; - let (mut parked, retired) = KBox::into_inner(pair).park(); - // This park follows the consumed release of `context`. - parked.release_consumed(context.scheduler_va()); - if linked { - // Guard (kill before free): the firmware may still walk this context's work - // list into the graph, so it is not freed or recycled before the firmware stops. - kernel::pr_warn_once!("G17: keeping render graphs of an unkilled context that still links work\n"); - keep_graph(&mut self.kept_graphs, retired); - } else { - *target = Some(retired); - } + let (parked, retired) = KBox::into_inner(pair).park(); + *target = Some(retired); entry.parked = Some(parked); // Detached backing retains these references until the worker runs. entry.context = None; @@ -163,12 +136,8 @@ impl RenderState { if let Some(id) = self.retired[index].buffer_id() { buffers.unreserve_owner(id)?; } - let retired = self.retired.remove(index).map_err(|_| EIO)?; - if linked { - keep_graph(&mut self.kept_graphs, retired); - } else { - *out.binding_slot()? = Some(retired); - } + let target = out.binding_slot()?; + *target = Some(self.retired.remove(index).map_err(|_| EIO)?); } Ok(()) } @@ -193,23 +162,3 @@ impl Registry { Ok(()) } } - -/// Graphs that could not be recorded in `kept_graphs` and were leaked instead, for debugfs. -pub(super) static LEAKED_GRAPHS: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); - -/// Keeps `graph` until the firmware stops; if even that cannot be recorded, it is leaked -/// rather than freed. -fn keep_graph( - kept: &mut KVec, - graph: crate::g17::queue::render::RetiredBinding, -) { - if kept.reserve(1, GFP_KERNEL).is_ok() { - if let Err(error) = kept.push_within_capacity(graph) { - LEAKED_GRAPHS.fetch_add(1, core::sync::atomic::Ordering::Relaxed); - core::mem::forget(error.0); - } - } else { - LEAKED_GRAPHS.fetch_add(1, core::sync::atomic::Ordering::Relaxed); - core::mem::forget(graph); - } -} diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 0222ae4c478c8..8a637e0b3ed01 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -100,8 +100,6 @@ impl ReclaimBatch { struct Service<'a> { render: &'a mut super::render::RenderState, - admission: &'a crate::g17::queue::admission::Pool, - objects: &'a Arc, compute: &'a [Option], buffers: &'a mut BufferIds, init: &'a InitData, @@ -117,8 +115,8 @@ impl Host for Service<'_> { self.compute .iter() .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) + .filter_map(|entry| entry.queue.as_ref()) + .all(|queue| queue.idle_for_context(context)) } fn flists_released(&self, context: &Arc) -> bool { self.render.teardown_pools_released(context) @@ -139,15 +137,8 @@ impl Host for Service<'_> { self.primary.notify(MSG_CONTROL_NOTIFY) } fn detach(&mut self, context: &Arc) -> Result { - let linked = context.retain_linked_work(self.objects); - let (_, _, slots) = self.render.exit_scope(context); - crate::g17::trace::teardown(5, u64::from(context.id()), slots, u64::from(linked)); self.render .detach_context(context, self.buffers, self.reclaim)?; - // The firmware consumed this context's scheduler release after its kill (if any) - // was acknowledged and its pairs went idle: it no longer names these pairs, so - // their kill hold ends here rather than at the next processor stop. - self.admission.release_killed_slots(slots); // detach_context preflights this slot together with every graph slot. // Retain it before Pending removes its reference under the device lock. self.reclaim.context(context) @@ -182,7 +173,6 @@ impl Firmware { self.queues.render.observe_teardown_releases(&self.init)?; let recovery = self.recovery.pending() || self.init.recovery_state()? != 0; let Registry { - admission, render, compute, buffers, @@ -191,8 +181,6 @@ impl Firmware { } = &mut self.queues; let mut service = Service { render, - admission, - objects: &self.pool, compute, buffers, init: &self.init, @@ -202,17 +190,10 @@ impl Firmware { teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); - let mut consumed = false; while teardown .take_consumed(consumer, producer, &mut service)? .is_some() - { - consumed = true; - } - if consumed { - // A channel whose last scheduler release is consumed can change owner. - self.primary.state.shared.channels.notify(); - } + {} Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index fa2a72a77f060..1a25cd7e3fa3b 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -24,45 +24,6 @@ struct Entry { phase: Phase, armed: u64, announce_pending: bool, - render_kill: Option, - /// Since when, and whether reported, a render-only kill has waited for other contexts' - /// compute to pause. - kill_wait: Option<(u64, bool)>, - /// A render-only kill of this context was refused (counted once per context). - kill_refused: bool, -} - -/// The kill of a closed context whose compute side is clean, published so that its render -/// pairs' executed but unstamped commands can be released (see `Pair::firmware_drained`). -/// The scheduler page's killed state, not this record, proves the kill to the pairs. -#[derive(Clone, Copy)] -struct RenderKill { - cookie: u64, - published: u64, - announced: bool, - acknowledged: bool, -} - -impl RenderKill { - fn new(cookie: u64, now: u64) -> Self { - Self { - cookie, - published: now, - announced: false, - acknowledged: false, - } - } - /// Only this kill's exact cookie acknowledges it, once. - fn acknowledge(&mut self, cookie: u64) -> bool { - if cookie == 0 || cookie != self.cookie || self.acknowledged { - return false; - } - self.acknowledged = true; - true - } - fn awaiting(&self, now: u64, window: u64) -> bool { - !self.acknowledged && now.saturating_sub(self.published) < window - } } /// Publication and announcement errors retain the entry; no timeout frees @@ -114,21 +75,6 @@ impl Pending { self.entries.is_empty() } - /// The context's scheduler release is armed or published, and the queues it - /// last ran have not been detached yet. Until then the firmware may still - /// process the release against those queues: none of them may change owner. - pub(crate) fn releasing(&self, context: &Arc) -> bool { - self.find(context) - .is_some_and(|index| self.entries[index].phase != Phase::Deferred) - } - - /// `releasing` for the context that owns scheduler page `scheduler`. - pub(crate) fn releasing_scheduler(&self, scheduler: u64) -> bool { - self.entries.iter().any(|entry| { - entry.phase != Phase::Deferred && entry.context.scheduler_va() == scheduler - }) - } - fn find(&self, context: &Arc) -> Option { self.entries .iter() @@ -143,9 +89,6 @@ impl Pending { phase, armed: now, announce_pending: false, - render_kill: None, - kill_wait: None, - kill_refused: false, }; self.entries.push(entry, GFP_KERNEL)?; Ok(()) @@ -275,94 +218,6 @@ impl Pending { Ok(None) } - /// A render-only kill of the closed context at `index` was published. - pub(crate) fn render_kill_published(&self, index: usize) -> bool { - self.entries.get(index).is_some_and(|entry| entry.render_kill.is_some()) - } - - /// Record a published render-only kill. Publication is irrevocable: the record stays - /// with the entry until the context's scheduler release is consumed. - pub(crate) fn record_render_kill(&mut self, index: usize, cookie: u64, now: u64) -> Result { - let entry = self.entries.get_mut(index).ok_or(EINVAL)?; - if entry.render_kill.is_some() { - return Err(EINVAL); - } - entry.render_kill = Some(RenderKill::new(cookie, now)); - Ok(()) - } - - /// Ring the doorbell for published render-only kills not yet announced. A failed - /// doorbell is retried on a later pass and never duplicates the record. - pub(crate) fn announce_render_kills(&mut self, mut notify: impl FnMut() -> Result) { - for entry in self.entries.iter_mut() { - if let Some(kill) = entry.render_kill.as_mut() { - if !kill.announced && !kill.acknowledged && notify().is_ok() { - kill.announced = true; - } - } - } - } - - /// Only the exact cookie of an unacknowledged render-only kill is accepted. - pub(crate) fn acknowledge_render_kill(&mut self, cookie: u64) -> bool { - self.entries - .iter_mut() - .filter_map(|entry| entry.render_kill.as_mut()) - .any(|kill| kill.acknowledge(cookie)) - } - - /// A render-only kill awaits its acknowledgement within the polling window. - pub(crate) fn render_kill_polling(&self, now: u64, window: u64) -> bool { - self.entries - .iter() - .any(|entry| entry.render_kill.is_some_and(|kill| kill.awaiting(now, window))) - } - - /// The render-only kill of the entry at `index` waits; how long, and whether this is the - /// first time it crossed `slow`. - pub(crate) fn note_kill_wait(&mut self, index: usize, now: u64, slow: u64) -> (u64, bool) { - let Some(entry) = self.entries.get_mut(index) else { - return (0, false); - }; - let (since, warned) = entry.kill_wait.get_or_insert((now, false)); - let waited = now.saturating_sub(*since); - let crossed = waited >= slow && !*warned; - *warned |= crossed; - (waited, crossed) - } - - /// The render-only kill was published: how long it waited, if it did. - pub(crate) fn take_kill_wait(&mut self, index: usize, now: u64) -> u64 { - self.entries - .get_mut(index) - .and_then(|entry| entry.kill_wait.take()) - .map_or(0, |(since, _)| now.saturating_sub(since)) - } - - /// A render-only kill of the context at `index` was refused: it no longer waits, and this - /// returns whether it is the first refusal for this context. - pub(crate) fn refuse_render_kill(&mut self, index: usize) -> bool { - let Some(entry) = self.entries.get_mut(index) else { - return false; - }; - entry.kill_wait = None; - !core::mem::replace(&mut entry.kill_refused, true) - } - - /// The context at `index` is not waiting for a render-only kill (any more). - pub(crate) fn clear_kill_wait(&mut self, index: usize) { - if let Some(entry) = self.entries.get_mut(index) { - entry.kill_wait = None; - } - } - - /// How long each waiting render-only kill has waited. - pub(crate) fn kill_waits(&self, now: u64) -> impl Iterator + '_ { - self.entries - .iter() - .filter_map(move |entry| entry.kill_wait.map(|(since, _)| now.saturating_sub(since))) - } - /// Poll for a bounded window at close and again at actual release publication. /// Failed announcements remain retryable on later service even after polling /// expires. Time passage never substitutes for a firmware-consumption witness. @@ -372,21 +227,3 @@ impl Pending { }) } } - -#[kernel::macros::kunit_tests(asahi_g17_render_kill)] -mod render_kill_tests { - use super::RenderKill; - - #[test] - fn only_the_exact_cookie_acknowledges_a_render_kill_once() { - let mut kill = RenderKill::new(7, 1_000); - assert!(kill.awaiting(1_000, 10)); - assert!(!kill.acknowledge(0)); - assert!(!kill.acknowledge(8)); - assert!(kill.awaiting(1_009, 10)); - assert!(!kill.awaiting(1_010, 10)); - assert!(kill.acknowledge(7)); - assert!(!kill.acknowledge(7)); - assert!(!kill.awaiting(1_001, 10)); - } -} diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index ce003e6b18196..e76dab07f56db 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -6,100 +6,11 @@ pub(crate) const MAX_RENEWALS: u8 = 8; -/// A borrowed render graph or queued off-lock retirement is not evidence of -/// progress. Share the finite deferral bound with cross-VM attribution. -pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS } - -/// Keep the scheduler poll, control-ring waits and ordinary jobs at two seconds. -/// Native ray traversal and AS builds can exceed one poll (CTS stress). Four -/// extra polls give it ten seconds, still one tenth of the legacy scheduler's -/// fallback. This is an execution allowance, not evidence of GPU progress, and -/// is consumed once per scheduler job, independently of cross-VM/replay grace. -pub(crate) const POLL_MS: u32 = 2000; -const EXECUTION_RENEWALS: u8 = 4; - -/// How long an innocent client's publication waits for a recovery to settle and its -/// spared work to drain, instead of failing with EBUSY: the watchdog defers work behind -/// another VM at most this long before it settles it (`classify`; ray work gets the -/// execution renewals on top). -pub(crate) const RECOVERY_WAIT_MS: u32 = - POLL_MS * (1 + EXECUTION_RENEWALS as u32 + MAX_RENEWALS as u32); - -/// A background control/replay poll has no scheduler fence to expire it. -/// Its timestamp belongs to the exact transaction and is never renewed by work. -pub(crate) fn background_stalled(since: Option, now: u64) -> bool { - since.is_some_and(|since| now.saturating_sub(since) >= 10_000_000_000) -} - -/// Input comes only from fully validated commands. Render metadata covers the -/// VDM half (vertex/tessellation/geometry) and the fragment half independently. -/// AS builders do not traverse rays and need an explicit, bounded UAPI hint. -pub(crate) fn select(compute_ray: bool, render_ray: [bool; 2], as_build: bool) -> bool { - compute_ray || render_ray.into_iter().any(|enabled| enabled) || as_build -} - -/// Firmware dependencies can hold an untagged helper behind a long command on -/// the other engine. Classify the complete validated ioctl before enqueueing -/// anything; no state or allowance leaks into a later submission. -pub(crate) fn select_submission(commands: impl Iterator) -> bool { - commands.into_iter().any(|enabled| enabled) -} - -/// Only the explicit AS-build bit is newly accepted; reserved and unrelated -/// flags remain invalid on G17P. Return None before any command is published. -pub(crate) fn as_build(flags: u32) -> Option { - const AS_BUILD: u32 = 1 << 2; - if flags & !AS_BUILD != 0 { None } else { Some(flags & AS_BUILD != 0) } -} - -pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) -> bool { - if !published || !enabled || *renewals >= EXECUTION_RENEWALS { - return false; - } - *renewals += 1; - true -} - -/// Polls a published job may be renewed while the GPU keeps retiring work. A job -/// shares the GPU's cores with every other runnable queue, so its wall time grows -/// with the other clients' work: beside four other clients a 1.5 s dispatch took up -/// to 5 s, and lower-priority work waits behind higher-priority work. It has not -/// hung while the GPU keeps completing commands. Fourteen renewals bound the wait at -/// thirty seconds, after which a job that still has not finished fails as before. -const SHARED_RENEWALS: u8 = 14; - -/// The GPU counts as live while some command retired this recently. Under heavy -/// sharing the longest gap between retirements measured 1.2 s; a GPU that retires -/// nothing for this long has stopped. -pub(crate) const LIVENESS_NS: u64 = 5_000_000_000; - -/// Whether some command retired within [`LIVENESS_NS`] (`since_retired`). -pub(crate) fn retiring(since_retired: Option) -> bool { - since_retired.is_some_and(|since| since < LIVENESS_NS) -} - -/// Renews a published job's deadline while the GPU is `shared`, at most -/// [`SHARED_RENEWALS`] times. It is shared while some command retires -/// ([`retiring`]), or while a long job of another queue runs beside a long job: -/// two ray traversal dispatches of about 6 s each took 10 to 17 s side by side, and -/// neither retires anything until it ends. -pub(crate) fn renew_shared(published: bool, shared: bool, renewals: &mut u8) -> bool { - if !published || !shared || *renewals >= SHARED_RENEWALS { - return false; - } - *renewals += 1; - true -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { - /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. - pub(crate) vm: u64, + pub(crate) vm: usize, pub(crate) published: u64, pub(crate) finished: bool, - /// Its queue already failed (quarantined) and holds the GPU only until the firmware drains - /// it. No later watchdog of its own will settle it. - pub(crate) failed: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -109,187 +20,23 @@ pub(crate) enum Decision { Spare, } -/// Another VM's older unfinished work that can block `own`: live work before work whose queue -/// already failed, the oldest of each first. -pub(crate) fn blocker(own: Head, others: impl Iterator) -> Option { - others - .filter(|other| other.vm != own.vm && !other.finished && other.published < own.published) - .min_by_key(|other| (other.failed, other.published)) -} - -/// Work behind another VM's older unfinished work is renewed up to MAX_RENEWALS times, so that -/// the blocker's own failure and teardown can free the GPU. After that it is spared only behind -/// a live blocker, which its own watchdog still settles. A blocker that already failed waits only -/// for the firmware to drain it and no longer excuses later work: otherwise an innocent frame -/// published before the culprit, which timed out first, would spare the culprit, and a pair that -/// never drains would spare every later timeout. -pub(crate) fn decide(own: Head, renewals: u8, blocker: Option) -> Decision { - match blocker { - _ if own.finished => Decision::Own, - None => Decision::Own, - Some(_) if renewals < MAX_RENEWALS => Decision::Defer, - Some(blocker) if blocker.failed => Decision::Own, - Some(_) => Decision::Spare, - } -} - -/// A watchdog's decision with the heads it rests on, for its log line. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Verdict { - pub(crate) decision: Decision, - pub(crate) own: Head, - pub(crate) blocker: Option, -} - -impl Verdict { - /// The blocker a watchdog line names: at the first deferral and once the grace has run out. - pub(crate) fn reported_blocker(&self, renewals: u8) -> Option { - self.blocker.filter(|_| renewals == 0 || self.decision != Decision::Defer) - } -} - -pub(crate) fn attribute(own: Head, renewals: u8, others: impl Iterator) -> Verdict { - let blocker = blocker(own, others); - Verdict { - decision: decide(own, renewals, blocker), - own, - blocker, +pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { + if own.finished || !others.into_iter().any(|other| { + other.vm != own.vm && !other.finished && other.published < own.published + }) { + Decision::Own + } else if renewals < MAX_RENEWALS { + Decision::Defer + } else { + Decision::Spare } } -#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_execution_allowance))] +#[cfg(test)] mod tests { use super::*; - #[test] - fn selects_compute_both_render_halves_and_builds() { - assert!(!select(false, [false; 2], false)); - assert!(select(true, [false; 2], false)); - assert!(select(false, [true, false], false)); - assert!(select(false, [false, true], false)); - assert!(select(false, [true; 2], false)); - assert!(select(false, [false; 2], true)); - } - #[test] - fn helpers_share_only_the_current_submission_allowance() { - assert!(select_submission([false, true, false].into_iter())); - assert!(select_submission([true, false].into_iter())); - assert!(select_submission([false, true].into_iter())); - assert!(!select_submission([false, false].into_iter())); - assert!(!select_submission([].into_iter())); - let mut helper = 0; - for _ in 0..4 { assert!(renew_execution(true, true, &mut helper)); } - assert!(!renew_execution(true, true, &mut helper)); - assert!(!renew_execution(true, false, &mut 0)); - } - #[test] - fn build_hint_rejects_reserved_and_unknown_flags() { - assert_eq!(as_build(0), Some(false)); - assert_eq!(as_build(1 << 2), Some(true)); - for bit in 0..32 { - if bit == 2 { continue; } - assert_eq!(as_build(1 << bit), None); - assert_eq!(as_build((1 << bit) | (1 << 2)), None); - } - assert_eq!(as_build(u32::MAX), None); - } - #[test] - fn borrowed_render_and_queued_retirement_cannot_defer_forever() { - for count in 0..=u8::MAX { - assert_eq!(transaction_grace(count), count < MAX_RENEWALS); - } - } - #[test] - fn background_control_and_replay_polls_have_an_absolute_deadline() { - let since = 42; - assert!(!background_stalled(None, u64::MAX)); - assert!(!background_stalled(Some(since), since - 1)); - assert!(!background_stalled(Some(since), since + 9_999_999_999)); - for _ in 0..1024 { - assert!(background_stalled(Some(since), since + 10_000_000_000)); - } - assert!(background_stalled(Some(since), u64::MAX)); - } - fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { - attribute(own, renewals, others).decision - } - fn head(vm: u64, published: u64, finished: bool) -> Head { - Head { vm, published, finished, failed: false } - } - fn failed(vm: u64, published: u64) -> Head { - Head { vm, published, finished: false, failed: true } - } - #[test] - fn ray_query_execution_is_bounded_at_ten_seconds() { - let mut renewals = 0; - for poll in 1..=4 { - assert!(renew_execution(true, true, &mut renewals)); - assert_eq!(renewals, poll); - } - for _ in 0..256 { - assert!(!renew_execution(true, true, &mut renewals)); - } - assert_eq!(POLL_MS * (u32::from(renewals) + 1), 10_000); - } - #[test] - fn ordinary_and_unpublished_jobs_keep_the_first_deadline() { - for (published, enabled) in [(false, false), (false, true), (true, false)] { - let mut renewals = 0; - assert!(!renew_execution(published, enabled, &mut renewals)); - assert_eq!(renewals, 0); - } - } - #[test] - fn ray_query_and_cross_vm_credits_are_independent_and_finite() { - let mut ray = 0; - let own = head(5, 101, false); - for _ in 0..4 { - assert!(renew_execution(true, true, &mut ray)); - } - for victim in 0..MAX_RENEWALS { - assert!(!renew_execution(true, true, &mut ray)); - assert_eq!(classify(own, victim, [head(7, 100, false)].into_iter()), Decision::Defer); - } - assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); - assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); - } - #[test] - fn a_live_gpu_renews_a_published_job_for_thirty_seconds() { - let mut renewals = 0; - for poll in 1..=14u8 { - assert!(renew_shared(true, retiring(Some(4_900_000_000)), &mut renewals)); - assert_eq!(renewals, poll); - } - assert!(!renew_shared(true, true, &mut renewals)); - assert_eq!(POLL_MS * (u32::from(renewals) + 1), 30_000); - } - #[test] - fn a_stopped_gpu_or_unpublished_job_keeps_the_deadline() { - let mut renewals = 0; - assert!(!renew_shared(true, retiring(Some(LIVENESS_NS)), &mut renewals)); - assert!(!renew_shared(true, retiring(None), &mut renewals)); - assert!(!renew_shared(false, true, &mut renewals)); - assert_eq!(renewals, 0); - } - #[test] - fn a_long_job_beside_another_shares_the_gpu_without_retirements() { - // Two traversal dispatches side by side: nothing retires, yet both progress. - let mut renewals = 0; - let long_job_beside = true; - let shared = retiring(Some(LIVENESS_NS + 1)) || long_job_beside; - for poll in 1..=14u8 { - assert!(renew_shared(true, shared, &mut renewals)); - assert_eq!(renewals, poll); - } - // The thirty-second bound still ends a pair that never finishes. - assert!(!renew_shared(true, true, &mut renewals)); - } - #[test] - fn exhausted_or_invalid_credit_never_wraps() { - for value in 4..=u8::MAX { - let mut renewals = value; - assert!(!renew_execution(true, true, &mut renewals)); - assert_eq!(renewals, value); - } + fn head(vm: usize, published: u64, finished: bool) -> Head { + Head { vm, published, finished } } #[test] fn earlier_other_vm_gets_bounded_grace() { @@ -305,53 +52,6 @@ mod tests { assert_eq!(classify(head(7, 100, false), 0, [head(5, 101, false)].into_iter()), Decision::Own); } #[test] - fn a_blocker_that_already_failed_defers_then_never_spares() { - // An innocent frame published before the culprit timed out first and failed; its - // quarantined pair keeps its head until the firmware drains it. - let culprit = head(7, 101, false); - for count in 0..MAX_RENEWALS { - assert_eq!(classify(culprit, count, [failed(5, 100)].into_iter()), Decision::Defer); - } - assert_eq!(classify(culprit, MAX_RENEWALS, [failed(5, 100)].into_iter()), Decision::Own); - assert_eq!(classify(culprit, u8::MAX, [failed(5, 100)].into_iter()), Decision::Own); - // A pair that never drains does not spare any later timeout either. - let drained_never = [failed(5, 100), failed(5, 102)]; - for own in [head(7, 105, false), head(9, 200, false)] { - assert_eq!(classify(own, MAX_RENEWALS, drained_never.into_iter()), Decision::Own); - } - } - #[test] - fn a_live_blocker_still_spares_beside_a_failed_one() { - let own = head(9, 103, false); - let others = [failed(5, 100), head(7, 101, false), failed(8, 102)]; - assert_eq!(blocker(own, others.into_iter()), Some(head(7, 101, false))); - assert_eq!(classify(own, MAX_RENEWALS, others.into_iter()), Decision::Spare); - // Failed or not, a blocker must be older, unfinished and of another VM. - let late = [failed(5, 104), failed(9, 100), Head { finished: true, ..failed(5, 100) }]; - assert_eq!(blocker(own, late.into_iter()), None); - assert_eq!(classify(own, 0, late.into_iter()), Decision::Own); - // Among failed blockers the oldest is reported. - let both_failed = [failed(8, 102), failed(5, 100)]; - assert_eq!(blocker(own, both_failed.into_iter()), Some(failed(5, 100))); - } - #[test] - fn the_watchdog_line_names_the_blocker_at_the_first_deferral_and_the_end() { - let own = head(9, 103, false); - let verdict = attribute(own, 0, [failed(5, 100)].into_iter()); - let expected = Verdict { decision: Decision::Defer, own, blocker: Some(failed(5, 100)) }; - assert_eq!(verdict, expected); - assert_eq!(verdict.reported_blocker(0), Some(failed(5, 100))); - assert_eq!(attribute(own, 1, [failed(5, 100)].into_iter()).reported_blocker(1), None); - let last = attribute(own, MAX_RENEWALS, [failed(5, 100)].into_iter()); - assert_eq!(last.decision, Decision::Own); - assert_eq!(last.reported_blocker(MAX_RENEWALS), Some(failed(5, 100))); - let spared = attribute(own, MAX_RENEWALS, [head(5, 100, false)].into_iter()); - assert_eq!(spared.decision, Decision::Spare); - assert_eq!(spared.reported_blocker(MAX_RENEWALS), Some(head(5, 100, false))); - // Nothing blocks: nothing to report. - assert_eq!(attribute(own, 0, [].into_iter()).reported_blocker(0), None); - } - #[test] fn same_vm_finished_and_later_work_do_not_block() { let own = head(5, 101, false); assert_eq!(classify(own, 0, [head(5, 100, false), head(7, 100, true), head(7, 101, false), head(7, 102, false)].into_iter()), Decision::Own); diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index b0f2861382ad7..da43c10cd3bef 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -8,41 +8,19 @@ use core::{cell::RefCell, ops::Range}; use kernel::prelude::*; const MAX_RANGES: usize = 48; -// The persistent proof must retain a deep queue's rotating command buffers. -// Keep the transient parser request set small to bound its stack frame. -const MAX_CACHED_RANGES: usize = 128; type Request = (u64, u64, u8); -/// Match the page footprint already checked by `Vm::covers_range_locked`. -fn pages((address, size, access): Request) -> Option { - if size == 0 || access == 0 || access & !3 != 0 { - return None; - } - let mask = mmu::UAT_PGSZ as u64 - 1; - let end = address.checked_add(size)?; - let end = end.checked_add(mask)? & !mask; - let start = address & !mask; - Some((start, end.checked_sub(start)?, access)) -} - -fn contains((start, size, access): Request, (other, length, needed): Request) -> bool { - access & needed == needed - && other >= start - && other.checked_add(length).is_some_and(|end| - start.checked_add(size).is_some_and(|limit| end <= limit)) -} - #[derive(Clone, Copy)] -struct Requests { - ranges: [Request; CAPACITY], +struct Requests { + ranges: [Request; MAX_RANGES], count: usize, overflow: bool, } -impl Requests { +impl Requests { const fn new() -> Self { Self { - ranges: [(0, 0, 0); CAPACITY], + ranges: [(0, 0, 0); MAX_RANGES], count: 0, overflow: false, } @@ -52,55 +30,24 @@ impl Requests { &self.ranges[..self.count] } - fn covers(&self, request: Request) -> bool { - self.as_slice().iter().any(|&range| contains(range, request)) - } - fn insert(&mut self, request: Request) { - let Some(mut request) = pages(request) else { - self.overflow = true; - return; - }; - if self.covers(request) { + if self.as_slice().contains(&request) { return; } - // Merge only equal permissions, so an adjacent writable page never - // grants write access to a read-only page. Restart after each merge - // because it can join a range visited earlier. - let mut index = 0; - while index < self.count { - let range = self.ranges[index]; - let end = request.0 + request.1; - let other_end = range.0 + range.1; - if contains(request, range) - || (request.2 == range.2 && request.0 <= other_end && range.0 <= end) - { - if request.2 == range.2 { - let start = request.0.min(range.0); - request = (start, end.max(other_end) - start, request.2); - } - self.count -= 1; - self.ranges[index] = self.ranges[self.count]; - index = 0; - } else { - index += 1; - } - } - if self.count == CAPACITY { + if self.count == MAX_RANGES { self.overflow = true; return; } self.ranges[self.count] = request; self.count += 1; } - } -/// One logical queue's bounded cache of validated page ranges and permissions. +/// One logical queue's bounded cache of exact address, size and permission tuples. /// Mapping mutations make the epoch unavailable until their final page-table write. pub(crate) struct Proof { epoch: Option, - requests: Requests, + requests: Requests, } impl Proof { @@ -117,7 +64,7 @@ impl Proof { && requests .as_slice() .iter() - .all(|&request| self.requests.covers(request)) + .all(|request| self.requests.as_slice().contains(request)) } fn record(&mut self, epoch: Option, requests: Requests) { @@ -132,10 +79,7 @@ impl Proof { } // An incomplete union cannot be reused; retain the latest complete set. self.epoch = if requests.overflow { None } else { epoch }; - self.requests = Requests::new(); - for &request in requests.as_slice() { - self.requests.insert(request); - } + self.requests = requests; } /// The VM-job guard must already pin the mappings for this command. Keep the @@ -146,11 +90,10 @@ impl Proof { vm: &mmu::Vm, range: Range, window: &UscWindow, - variant: &crate::g17::fw::Variant, payload: &Payload, ) -> Result { let requests = RefCell::new(Requests::new()); - let optimistic = payload.validate(window, variant, &Space::Record(&range, &requests)); + let optimistic = payload.validate(window, &Space::Record(&range, &requests)); let requests = requests.into_inner(); if optimistic.is_ok() && !requests.overflow { if self.covers(vm.mapping_validation_epoch(), &requests) { @@ -164,7 +107,7 @@ impl Proof { self.epoch = None; } // Preserve per-range validation's failure and concurrent-mutation behavior. - payload.validate(window, variant, &Space::Checked(&range, vm)) + payload.validate(window, &Space::Checked(&range, vm)) } } @@ -200,101 +143,3 @@ impl AddressSpace for Space<'_> { } } } - - -#[kunit_tests(asahi_g17_validation_pages)] -mod tests { - use super::*; - const PAGE: u64 = mmu::UAT_PGSZ as u64; - - fn requests(range: Request) -> Requests { - let mut result = Requests::new(); - result.insert(range); - result - } - - #[test] - fn different_offsets_share_the_already_validated_page() { - let mut proof = Proof::new(); - proof.record(Some(7), requests((PAGE + 4, 4, 3))); - assert!(proof.covers(Some(7), &requests((PAGE + 0x100, 8, 1)))); - assert!(proof.covers(Some(7), &requests((2 * PAGE - 8, 8, 2)))); - assert!(!proof.covers(Some(7), &requests((2 * PAGE - 8, 16, 1)))); - assert!(!proof.covers(Some(8), &requests((PAGE + 4, 4, 1)))); - assert!(!proof.covers(None, &requests((PAGE + 4, 4, 1)))); - } - - #[test] - fn adjacent_equal_permissions_merge_but_holes_do_not() { - let mut ranges = requests((PAGE, 1, 1)); - ranges.insert((3 * PAGE, 1, 1)); - assert_eq!(ranges.count, 2); - assert!(!ranges.covers((2 * PAGE, PAGE, 1))); - ranges.insert((2 * PAGE, 1, 1)); - assert_eq!(ranges.count, 1); - assert_eq!(ranges.as_slice(), &[(PAGE, 3 * PAGE, 1)]); - } - - #[test] - fn adjacent_permissions_never_grant_access_to_another_page() { - let mut proof = Proof::new(); - let mut ranges = requests((PAGE, 1, 1)); - ranges.insert((2 * PAGE, 1, 3)); - proof.record(Some(1), ranges); - assert!(!proof.covers(Some(1), &requests((PAGE, 1, 2)))); - assert!(proof.covers(Some(1), &requests((2 * PAGE, 1, 2)))); - assert!(!proof.covers(Some(1), &requests((PAGE, 2 * PAGE, 3)))); - } - - #[test] - fn stronger_coverage_removes_only_fully_covered_ranges() { - let mut ranges = requests((PAGE, PAGE, 1)); - ranges.insert((2 * PAGE, PAGE, 2)); - ranges.insert((PAGE, 2 * PAGE, 3)); - assert_eq!(ranges.as_slice(), &[(PAGE, 2 * PAGE, 3)]); - assert!(!ranges.covers((0, PAGE, 1))); - } - - #[test] - fn persistent_cache_retains_a_deep_rotating_working_set() { - let mut proof = Proof::new(); - for index in 0..64 { - proof.record(Some(7), requests((index * 2 * PAGE, 8, 1))); - } - assert_eq!(proof.requests.count, 64); - for index in 0..64 { - assert!(proof.covers(Some(7), &requests((index * 2 * PAGE + 16, 8, 1)))); - } - assert_eq!(core::mem::size_of::(), core::mem::size_of::>()); - } - - #[test] - fn persistent_overflow_keeps_only_a_new_complete_proof() { - let mut proof = Proof::new(); - for index in 0..=MAX_CACHED_RANGES { - proof.record(Some(7), requests((index as u64 * 2 * PAGE, 8, 1))); - } - assert_eq!(proof.requests.count, 1); - assert!(!proof.covers(Some(7), &requests((0, 8, 1)))); - assert!(proof.covers(Some(7), - &requests((MAX_CACHED_RANGES as u64 * 2 * PAGE, 8, 1)))); - } - - #[test] - fn invalid_rounding_and_full_cache_disable_reuse() { - for range in [(0, 0, 1), (u64::MAX - 1, 1, 1), (0, 1, 0), (0, 1, 4)] { - assert!(pages(range).is_none()); - let mut proof = Proof::new(); - proof.record(Some(1), requests(range)); - assert!(!proof.covers(Some(1), &requests((0, 1, 1)))); - } - let mut ranges = Requests::new(); - for index in 0..=MAX_RANGES { - ranges.insert(((2 * index) as u64 * PAGE, PAGE, 1)); - } - assert!(ranges.overflow); - let mut proof = Proof::new(); - proof.record(Some(1), ranges); - assert!(!proof.covers(Some(1), &requests((0, PAGE, 1)))); - } -} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index f100820a0c75e..3bcf8a9692dcc 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -21,10 +21,8 @@ use kernel::{ }, // }, error::Result, - new_mutex, prelude::*, sync::aref::ARef, - sync::Mutex, uapi, // }; @@ -41,7 +39,8 @@ use crate::{ AsahiDrmDevice, AsahiDrmDriver, // }, - file, memacct, mmu, + file, + mmu, }; const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; @@ -61,11 +60,6 @@ pub(crate) struct AsahiObject { kernel: bool, /// Driver aliases outside the user GPUVA tree; they pin the backing. kernel_mappings: AtomicU64, - /// Memory accounting and process attribution of the held backing. - charge: memacct::Charge, - /// Serializes overlapping timestamp writes through every binding of this BO. - #[pin] - pub(crate) timestamp_lock: Mutex<()>, } /// Type alias for the shmem GEM object type for this driver. @@ -80,9 +74,6 @@ pub(crate) type Object = shmem::Object; /// comes with unevictable, as `mapping_set_inaccessible()` sets it; nothing reads or writes a GEM /// shmem file through its mapping. Releasing idle backing or purging clears the mark, so the /// released pages can be swapped and migrated again. -/// -/// The same transitions charge and uncharge the held pages (see [`memacct`]): a held user object -/// counts towards the resident set of the process that created it. pub(crate) fn set_mapping_held(gem: &Object, held: bool) { let raw = gem.as_raw(); // SAFETY: `gem` keeps the GEM object alive. A shmem-backed object's `filp` and its @@ -108,7 +99,6 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { } else { flags.fetch_and(!inaccessible, Ordering::Relaxed); } - gem.charge.set_held(held); } /// Takes the SG table of `gem`, which holds its pages for the device, and marks the @@ -116,7 +106,7 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { pub(crate) fn held_sg_table<'a>( gem: &'a Object, dev: &'a kernel::device::Device, -) -> Result> { +) -> Result<&'a kernel::scatterlist::SGTable> { let sgt = gem.sg_table(dev)?; set_mapping_held(gem, true); Ok(sgt) @@ -125,12 +115,17 @@ pub(crate) fn held_sg_table<'a>( /// VM-private BOs share their assigned VM's reservation object. Validate this /// before acquiring mapping state; generic GPUVM accepts external shareable BOs. pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { - if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 && vm.is_extobj(gem) { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(gem) + { return Err(EINVAL); } Ok(()) } +unsafe impl Send for AsahiObject {} +unsafe impl Sync for AsahiObject {} + /// A shared reference to a GEM object for this driver. pub(crate) struct ObjectRef { /// The underlying GEM object reference @@ -216,7 +211,6 @@ pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, kernel: bool, - charge: memacct::Charge, } /// Create a new kernel-owned GEM object. @@ -235,23 +229,20 @@ pub(crate) fn new_kernel_object_wc(dev: &AsahiDrmDevice, size: usize) -> Result< /// Validate before rounding: wrapping a user size can create a much smaller /// backing than requested (or panic with overflow checks). Kernel callers use /// the same boundary so malformed calculated sizes cannot bypass this check. -pub(crate) fn checked_object_size(size: usize) -> Result { - if size == 0 { - return Err(EINVAL); - } +fn checked_object_size(size: usize) -> Result { + if size == 0 { return Err(EINVAL); } size.checked_add(mmu::UAT_PGMSK) .map(|rounded| rounded & !mmu::UAT_PGMSK) .ok_or(EOVERFLOW) } + fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> Result { - let size = checked_object_size(size)?; let gem = shmem::Object::::new( dev, - size, + checked_object_size(size)?, shmem::ObjectConfig:: { reclaimable_cpu_mappings: false, - pages_no_reclaim: false, map_wc, parent_resv_obj: None, }, @@ -259,7 +250,6 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> flags: 0, exportable: false, kernel: true, - charge: memacct::Charge::kernel(&memacct::LEDGER, size as u64), }, )?; @@ -269,17 +259,12 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> } /// Create a new user-owned GEM object with the given flags. -/// -/// `commit` is the admission of `size` (already rounded by [`checked_object_size`]) for the -/// creating client; the object is attributed to the current process. pub(crate) fn new_object( dev: &AsahiDrmDevice, size: usize, flags: u32, parent_object: Option<&shmem::Object>, reclaim_cpu: bool, - pages_no_reclaim: bool, - commit: memacct::Commit, ) -> Result> { if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() { @@ -291,7 +276,6 @@ pub(crate) fn new_object( checked_object_size(size)?, shmem::ObjectConfig:: { reclaimable_cpu_mappings: parent_object.is_some() && reclaim_cpu, - pages_no_reclaim, map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, parent_resv_obj: parent_object, }, @@ -299,7 +283,6 @@ pub(crate) fn new_object( flags, exportable: parent_object.is_none(), kernel: false, - charge: memacct::Charge::user_current(commit), }, )?; @@ -325,22 +308,16 @@ impl DriverObject for AsahiObject { exportable: args.exportable, kernel: args.kernel, kernel_mappings: AtomicU64::new(0), - charge: args.charge, - timestamp_lock <- new_mutex!((), "Asahi timestamp stores"), }) } /// Objects the DRM core allocates for us (dma-bufs imported from other devices and dumb /// buffers) are shared user objects, like a GEM_CREATE object without flags. - /// - /// Their memory is not counted: an import's backing belongs to, and is charged by, its - /// exporter, and the driver offers no dumb buffers (it has no KMS). fn core_object_args(_dev: &AsahiDrmDevice, _size: usize) -> Result { Ok(AsahiObjConfig { flags: 0, exportable: true, kernel: false, - charge: memacct::Charge::none(), }) } @@ -395,33 +372,11 @@ impl Drop for KernelMappingPin { } impl AsahiObject { - /// Whether CPU mappings use the cacheable WRITEBACK policy. - pub(crate) fn is_writeback(&self) -> bool { - self.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK != 0 - } - pub(crate) fn idle_reclaim_candidate(&self) -> bool { !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 } } -/// Populates and pins a new user object through its DMA mapping, before any CPU view exists. -/// -/// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the -/// freshly zeroed pages to the point of coherency, so that no dirty line can later overwrite data -/// written through a write-combined view. The pin keeps reclaim from creating a new cacheable -/// alias. Only used when the GPU asks for it ([`crate::gpu::Gpu::prepare_new_objects`]). -/// -pub(crate) fn prepare_new_object( - gem: &Object, - dev: &kernel::device::Device, -) -> Result { - // The native backend requires a non-coherent DMA device. Use its DMA mapping - // rather than interpreting DMA addresses as CPU physical addresses. - let _sgt = held_sg_table(gem, dev)?; - Ok(()) -} - /// Purge advice for a VM-private, non-exportable object. Returns whether the /// contents are retained (0 once purged). pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { @@ -431,26 +386,6 @@ pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { Ok(u32::from(gem.madvise(i32::from(dontneed)))) } -/// Large user BOs have a cheap ENOMEM fallback instead of per-folio reclaim. -/// Keep ordinary resource allocations and kernel-owned backing on their existing -/// policy. 256 MiB also matches Mesa's maximum reusable BO-cache footprint. -pub(crate) fn user_pages_no_reclaim(size: usize, g17: bool) -> bool { - g17 && size >= 256 * 1024 * 1024 -} - -#[kernel::macros::kunit_tests(asahi_gem_allocation_policy)] -mod allocation_policy_tests { - use super::*; - - #[test] - fn large_user_backing() { - assert!(!user_pages_no_reclaim(256 * 1024 * 1024 - 1, true)); - assert!(user_pages_no_reclaim(256 * 1024 * 1024, true)); - assert!(user_pages_no_reclaim(3_257_113_344, true)); - assert!(!user_pages_no_reclaim(3_257_113_344, false)); - } -} - /// Populates and pins a new user object through its DMA mapping, before any CPU view exists. /// /// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the @@ -461,7 +396,7 @@ mod allocation_policy_tests { /// The DMA mapping only cleans if the device is not marked DMA-coherent (G15's synthesized node /// has no `dma-coherent`). Should it ever be marked so, the pages are cleaned here instead. #[cfg(CONFIG_DRM_ASAHI_G15)] -pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result { +pub(crate) fn prepare_new_object(gem: &Object) -> Result { // SAFETY: G15 objects belong to the retained bound platform device; user object // creation occurs only after registration and before a file can outlive its device. let dev: &kernel::device::Device = gem.dev().as_ref().as_ref(); @@ -498,7 +433,9 @@ pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result { Ok(()) } + #[cfg(CONFIG_DRM_ASAHI_G15)] -extern "C" { - fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; -} +extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } + +#[cfg(not(CONFIG_DRM_ASAHI_G15))] +pub(crate) fn prepare_new_object(_gem: &Object) -> Result { Err(EINVAL) } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 3da16c10696e3..a06b5da5ed6b1 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,6 +11,7 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. +use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -35,7 +36,6 @@ use kernel::{ }, macros::versions, new_mutex, - new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -45,7 +45,6 @@ use kernel::{ }, Arc, Mutex, - SpinLock, UniqueArc, // }, time::{ @@ -86,7 +85,6 @@ use crate::{ queue, regs, stats, - stats_raw, workqueue, // }; @@ -261,30 +259,18 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, - /// Last raw firmware statistics of this device, exported read-only - /// through `stats_raw`. - raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, + #[allow(clippy::vec_box)] #[pin] - garbage_contexts: Mutex>, + garbage_contexts: Mutex>>>, } /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { - /// Admission bound checked before copying the userspace command stream. - fn max_cmdbuf_size(&self) -> usize { - usize::MAX - } - /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; - /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. - fn raw_stats_snapshot(&self) -> Option>> { - None - } - /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -299,10 +285,6 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; - /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). - fn supports_soft_faults(&self) -> bool { - crate::module_parameters::fault_control.value() == 0xb - } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false @@ -315,14 +297,14 @@ pub(crate) trait Gpu: Send + Sync { fn feature_flags(&self) -> u64 { let mut flags = 0; if self.supports_vm_status() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR); + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; } if self.queue_limits().is_some() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES); + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; } - if self.soft_faults() && self.supports_soft_faults() { - flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); + if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } flags } @@ -330,38 +312,35 @@ pub(crate) trait Gpu: Send + Sync { fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None } - /// The VA range that user VMs may use (`vm_start..vm_end` of GET_PARAMS). + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; + /// Backend-specific user address aperture; other GPUs keep their geometry's range. fn user_range(&self) -> Range { self.uat_geometry().user_usable_range() } - /// Whether this GPU uses the tracked submission model of `file.rs` (besides G17, which - /// `file.rs` recognizes by type): VM_BIND batches are validated as a whole against - /// driver-reserved ranges before any change, and SUBMIT copies the command buffer before - /// resolving syncs because the queue installs a fence in every output sync even on failure. + /// Use tracked command publication and sticky VM status on this backend. fn tracked_submission(&self) -> bool { false } - /// Whether SOFT_FAULTS may be advertised when `fault_control` selects soft faults. + /// Whether faults may be reported as recoverable to userspace. fn soft_faults(&self) -> bool { true } - /// Whether new user objects must be populated, cleaned to the point of coherency and pinned - /// at creation, before any CPU view exists (GPUs whose device is not DMA-coherent and whose - /// default CPU mappings are write-combined). + /// Populate and clean fresh object backing before any WC CPU mapping. fn prepare_new_objects(&self) -> bool { false } - /// The largest VM_BIND batch, if the GPU's UAPI contract bounds it; with a bound, a batch also - /// needs at least one operation, a non-null pointer and a stride that is a multiple of 8 and at - /// most a page (EINVAL otherwise). `None` keeps the shared rules. + /// Optional backend-specific upper bound for a VM_BIND batch. fn bind_batch_limit(&self) -> Option { None } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; - /// Admit a client's parameter ABI before creating its VM. + /// A userspace VM handle was closed; release backend-only registry references. + fn vm_closed(&self) {} + /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS + /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 + /// T8122 runtime admits any client. fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { Ok(()) } @@ -371,13 +350,6 @@ pub(crate) trait Gpu: Send + Sync { } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; - /// Create a VM charged to a DRM file when the backend bounds retained bindings. - fn new_vm_for_file(&self, kernel_range: Range, _file_id: u64) -> Result { - self.new_vm(kernel_range) - } - /// Release the backend's handle owner after closing user mapping admission. - /// Accepted jobs and firmware resource leases retain their own references. - fn vm_closed(&self, _vm: &mmu::Vm) {} /// Create a new user command queue. fn new_queue( &self, @@ -386,14 +358,13 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result>; - /// Bind a BO range as a firmware- or CPU-written timestamp buffer. + /// Map a BO as a timestamp buffer fn map_timestamp_buffer( &self, bo: gem::ObjectRef, range: Range, - ) -> Result; + ) -> Result; } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -435,8 +406,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_cfg(&self) -> &'static hw::HwConfig; /// Get the dynamic GPU configuration for this SoC. fn get_dyncfg(&self) -> &hw::DynConfig; - /// Register an unused context, with the objects it retains, for invalidation. - fn free_context(&self, ctx: workqueue::RetiredContext); + /// Register an unused context as garbage + fn free_context(&self, data: KBox>); } /// Private generic trait for functions that don't need to escape this module. @@ -781,7 +752,6 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, - mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) @@ -846,12 +816,6 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); - // Per-device raw statistics snapshot, shared with the debugfs export. - let raw_stats_snap = Arc::pin_init( - new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), - GFP_KERNEL, - )?; - let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -867,7 +831,6 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, - raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -895,7 +858,6 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), - raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1127,7 +1089,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(u64::from(map.writable)), + readwrite: U64(map.writable as u64), }; }); @@ -1177,14 +1139,20 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let guard = self.dev.registration_guard()?; + let Some(guard) = self.dev.registration_guard() else { + return None; + }; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if let Some(fault) = &info { - dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); + if info.is_some() { + dev_err!( + self.dev.as_ref(), + " Fault info: {:#x?}\n", + info.as_ref().unwrap() + ); } info }) @@ -1351,10 +1319,6 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } - fn raw_stats_snapshot(&self) -> Option>> { - Some(self.raw_stats.clone()) - } - fn as_any(&self) -> &dyn Any { self } @@ -1437,7 +1401,6 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1455,7 +1418,6 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, - usage, )?, GFP_KERNEL, )?) @@ -1465,15 +1427,15 @@ impl Gpu for GpuManager::ver { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - crate::timestamp::Buffer::firmware(bo.map_range_into_range( + ) -> Result { + bo.map_range_into_range( self.uat.kernel_vm(), range, IOVA_KERN_TIMESTAMP_RANGE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - )?) + ) } } @@ -1498,7 +1460,7 @@ impl GpuManager for GpuManager::ver { core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); for ctx in garbage_ctx { - if self.invalidate_context(&ctx.data).is_err() { + if self.invalidate_context(&ctx).is_err() { dev_err!( self.dev.as_ref(), "GpuContext: Failed to invalidate GPU context!\n" @@ -1507,9 +1469,6 @@ impl GpuManager for GpuManager::ver { panic!("GPU firmware timed out"); } } - // The firmware no longer knows the context's queues, so their structures may go now, - // ahead of the context data they point to. - core::mem::drop(ctx.retained); } let mut guard = self.alloc.lock(); @@ -1778,7 +1737,7 @@ impl GpuManager for GpuManager::ver { &self.dyncfg } - fn free_context(&self, ctx: workqueue::RetiredContext) { + fn free_context(&self, ctx: KBox>) { let mut garbage = self.garbage_contexts.lock(); if garbage.push(ctx, GFP_KERNEL).is_err() { diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 2650131a4a293..deffe3fe47d0b 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -13,7 +13,6 @@ const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; pub(crate) mod t602x; -pub(crate) mod t6040; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; @@ -24,7 +23,6 @@ pub(crate) mod t8140; pub(crate) enum GpuGen { G13 = 13, G14 = 14, - G16 = 16, G17 = 17, } diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs deleted file mode 100644 index c2e7c06f67b7f..0000000000000 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ /dev/null @@ -1,216 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Hardware configuration for t6040 platforms (M4 Pro). -//! -//! The M4 Pro runs the macOS 26.6 GPU firmware (25G83), which has the A18 Pro firmware's -//! interface; [`fw::variant::V25G83`] describes where it differs. - -use crate::g17::fw::{ - self, - initdata::{PerfConfig, RegionCConfig, RegisterWindow}, -}; -use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; -use crate::hw::{GpuGen, GpuVariant}; -use crate::mmu::UatQuirks; - -/// Static configuration of the T6040 GPU. -pub(crate) const CONFIG: Config = Config { - chip_id: CHIP_ID, - identity: Identity { - family: 0xa, - // ID register 0x0a03_1100: family 0xa, variant 3, revision 0x11 (B1). - variant: 3, - num_dies: 1, - gpu_variant: GpuVariant::S, - usc_generation: 3, - hal_generation: 200, - }, - uat_ias: 42, - uat_oas: 42, - // One coprocessor carries both firmware roles; `pagetables` holds only its root. - secondary_root_offset: None, - crash_buffers: CrashBuffers::FirmwareData, - // Read by the firmware through the kernel lower root, whose top-level entries it only - // samples when it starts. - pm_metrics: FixedBuffer { - va: 0x10_0008_0000, - size: 0x8000, - }, - windows: ®ISTER_WINDOWS, - reserved_slots: &[], - perf: &PERF, - opps: OPPS, - // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The - // rest is the policy the M4 Pro has run with; none of it was tested otherwise. - uat_quirks: UatQuirks { - reserved_tables: true, - clean_tables: true, - page_tlbi: true, - handoff_lock: false, - handoff_slot: 0xffff_ffff, - native_gpu_permissions: true, - }, - region_c: ®ION_C, - adopt_root_entries: false, - fabric_bridge: false, - // The firmware's per-slot recovery keys are not known for this build; recovery relies on - // the firmware's own blame. - recovery_slot_keys: None, - // Userspace drives this GPU as a G16 part. - gpu_generation: GpuGen::G16, - // The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 GPU firmware. - firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { - os_version: [26, 6, 2], - name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", - images: &[ - crate::fwid::FirmwareBuild { - name: "Apple GFX 25G83 for T6040 (normalized loaded image)", - text_size: 0x60000, - data_size: 0x138000, - entropy: [0x59839, 0x5991c], - text_sha256: [ - 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, - 0xaa, 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, - 0x10, 0xe0, 0x63, 0x87, 0x6d, 0xcd, - ], - }, - crate::fwid::FirmwareBuild { - name: "Apple GFX 25G83 for T6040 (normalized loaded image, alternate J616s fill)", - text_size: 0x60000, - data_size: 0x138000, - entropy: [0x59839, 0x5991c], - text_sha256: [ - 0x19, 0x0c, 0xef, 0x19, 0x46, 0xa2, 0x3d, 0xfb, 0xde, 0x18, 0x05, 0x92, 0x62, - 0x96, 0x31, 0xc3, 0xb2, 0x8d, 0x05, 0x09, 0xa1, 0x7e, 0xb7, 0x61, 0x08, 0xb7, - 0xc9, 0xa5, 0x2a, 0xcc, 0xd9, 0xac, - ], - }, - crate::fwid::FirmwareBuild { - name: "Apple GFX 25G83 for T6040 (normalized loaded image, second alternate J616s fill)", - text_size: 0x60000, - data_size: 0x138000, - entropy: [0x59839, 0x5991c], - text_sha256: [ - 0xd7, 0x25, 0x26, 0x4d, 0x48, 0x31, 0xb9, 0x5a, 0x5b, 0x8a, 0x37, 0x5a, 0xbf, - 0x2f, 0xb5, 0xcd, 0xfb, 0xd6, 0xd7, 0xcb, 0xc7, 0xf1, 0xba, 0xa2, 0xcc, 0x8b, - 0x37, 0xee, 0x5d, 0x6e, 0x5e, 0x98, - ], - }, - ], - variant: &fw::variant::V25G83, - }]), - // Measured on the 20-core M4 Pro; the alternative below remains experimental. - core_layout: CoreLayout { - cores: Some((2, 20)), - // Admit two clusters of ten slots with eight enabled cores in each cluster. - // Reusing the measured 20-core buffers on it remains experimental. - alternate_core_masks: Some((10, 8)), - operand_state: fw::compute::OperandState::TWO_BY_TEN, - deflake: fw::render::Deflake::TWO_CLUSTERS, - }, -}; - -/// Chip identifier written to the hardware data. -pub(crate) const CHIP_ID: u32 = 0x6040; - -/// Base of the firmware MMIO aperture in the kernel window. -const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; - -const fn window( - slot: usize, - phys: u64, - offset: u64, - size: u32, - unk_18: u64, - unk_20: u32, -) -> RegisterWindow { - RegisterWindow { - slot, - phys, - va: MMIO_APERTURE + offset, - size, - unk_18, - unk_20, - instances: 1, - phys_stride: 0, - } -} - -/// `window` repeated `instances` times, `phys_stride` apart in physical memory. -const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> RegisterWindow { - RegisterWindow { - instances, - phys_stride, - ..window - } -} - -/// Register windows the firmware maps, by register-map slot. Slot 3 holds the four memory -/// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. -pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ - window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), - instances( - window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), - 4, - 0x200_0000, - ), - window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), - window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), - window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), - window(11, 0x5_003d_8000, 0x28_4000, 0x4000, 0, 2), - window(12, 0x4_8565_c000, 0x28_c000, 0x4000, 0, 2), - window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), - window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), - window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), - instances( - window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), - 4, - 0x20_0000, - ), - window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), - window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), - window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), - window(29, 0x2_88d1_0000, 0x40_0000, 0x4000, 0xd1_0000, 2), - window(31, 0x2_88d4_0000, 0x40_8000, 0x4000, 0xd4_0000, 2), - window(32, 0x2_88d6_0000, 0x41_0000, 0x2_0000, 0xd6_0000, 2), - window(33, 0x2_88d4_4000, 0x43_4000, 0x4000, 0xd4_4000, 2), - window(34, 0x2_88d4_c000, 0x43_c000, 0x200, 0xd4_c000, 2), - window(35, 0x2_88e0_0000, 0x44_4000, 0x4000, 0xe0_0000, 2), - window(37, 0x2_88e4_0000, 0x44_c000, 0x4000, 0xe4_0000, 2), - window(39, 0x2_88e0_8000, 0x45_4000, 0x8000, 0, 2), - window(40, 0x2_88e1_c000, 0x46_0000, 0x4000, 0xe1_c000, 2), - window(41, 0x2_88e1_f800, 0x46_b800, 0x4000, 0, 2), - window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), -]; - -/// Inert defaults. The 25G83 builder requires the loader's per-chip performance maps. -pub(crate) const PERF: PerfConfig = PerfConfig { - states_a: [0; 11], - states_b: [0; 11], - relative_a: [0; 11], - relative_b: [0; 11], - sram_min_mv: 810, -}; - -/// Operating points the performance tables reference. -pub(crate) const OPPS: usize = 16; - -/// Region C. -const REGION_C: RegionCConfig = RegionCConfig { - unk_54: 0x0fa0, - unk_78: [1, 0, 0], - max_power_mw: 0, - unk_9c: 0, - unk_a0: 0, - unk_7c4: 1500, - unk_9e0: [ - 1, - 0x0102_0200, - 0x010a_2801, - 0x2802_0101, - 0x0101_010a, - 0x010a_2802, - 0x2801_0101, - 0x0001_010a, - ], -}; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index ea0f9cd782e81..143ba720d06f2 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -4,11 +4,10 @@ use crate::g17::{ Config, - CrashBuffers, FixedBuffer, Identity, // }; -use crate::hw::{GpuGen, GpuVariant}; +use crate::hw::GpuVariant; /// Static configuration of the T8140 GPU. pub(crate) const CONFIG: Config = Config { @@ -24,10 +23,10 @@ pub(crate) const CONFIG: Config = Config { uat_ias: 42, uat_oas: 42, // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. - secondary_root_offset: Some(0x40000), + secondary_root_offset: 0x40000, // The coprocessors request their crash log buffers at these addresses, in this order, and // expect them to be mapped before they start. - crash_buffers: CrashBuffers::Fixed([ + crash_buffers: [ FixedBuffer { va: 0x01e1_c000, size: 0x8000, @@ -36,31 +35,13 @@ pub(crate) const CONFIG: Config = Config { va: 0x01f5_0000, size: 0x8000, }, - ]), + ], // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. pm_metrics: FixedBuffer { va: 0x10_0008_0000, size: 0x8000, }, - recovery_slot_keys: Some(recovery::SLOT_KEYS), - gpu_generation: GpuGen::G17, - windows: ®ISTER_WINDOWS, - reserved_slots: &RESERVED_REGISTER_SLOTS, - perf: &PERF, - opps: OPPS, - uat_quirks: crate::mmu::UatQuirks::DEFAULT, - region_c: &crate::g17::fw::initdata::REGION_C_26A428, - adopt_root_entries: true, - fabric_bridge: true, - firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), - // The layouts the A18 Pro has always run with. - core_layout: crate::g17::CoreLayout { - cores: None, - alternate_core_masks: None, - operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, - deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, - }, }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; @@ -86,8 +67,6 @@ const fn window( size, unk_18, unk_20, - instances: 1, - phys_stride: 0, } } @@ -294,7 +273,7 @@ pub(crate) mod resources { pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = TVB_PAGE as usize; + pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; pub(crate) const TPC_MIN: usize = 0x3_0000; pub(crate) const TPC_MAX: usize = 0x200_0000; @@ -405,9 +384,6 @@ pub(crate) mod qos { pub(crate) mod scheduling { use crate::g17::fw::queue::Policy; - /// `DRM_ASAHI_PRIORITY_HIGH`: class 1, as the G13 tables map HIGH; the - /// policy word of the user classes is kept (no macOS trace of class 1 yet). - pub(crate) const HIGH: Policy = Policy::new(1, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_MEDIUM`. pub(crate) const MEDIUM: Policy = Policy::new(2, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_LOW`. @@ -417,11 +393,8 @@ pub(crate) mod scheduling { /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; - // One physical pair per logical owner leaves room for every execution - // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH, - // as many as a pair has command records (the render memory's RECORDS). - pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; - pub(crate) const RENDER_DEPTH: u8 = 32; + pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; + pub(crate) const RENDER_DEPTH: u8 = 16; pub(crate) const COMPUTE_DEPTH: usize = 16; } diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index ffc57c77afcb2..ca16e18f10fbf 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -43,9 +43,6 @@ pub(crate) struct Packet { pub(crate) wide_visibility:KVec, completion:UserFence, finish_claimed: AtomicBool, - /// Set before a batch with any of this packet's commands can change firmware-visible state, - /// and never cleared. - published: AtomicBool, /// Queue urgency (3 = UAPI realtime .. 0 = UAPI low): more urgent packets are published first. pub(crate) urgency:u8, /// Who submitted the packet, for the failure log only. @@ -81,10 +78,6 @@ impl Packet { } /// Called only after all published commands retired or before any publication. pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} - /// Called before the first firmware-visible change of a batch that includes this packet. - pub(crate) fn mark_published(&self) {self.published.store(true,Ordering::Relaxed);} - /// Whether the firmware may have seen any of this packet's commands. - pub(crate) fn may_be_published(&self)->bool {self.published.load(Ordering::Relaxed)} pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} pub(crate) fn finish(&self,result:Result) { // Retirement, cancellation and a queued scheduler timeout can race. @@ -239,7 +232,7 @@ impl queue::Queue for Queue { parser.finish().map_err(|_|EINVAL)?; let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, completion:self.fences.new_fence(0,Completion)?.into(), - finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), + finish_claimed:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; let mut job=self.entity.new_job(1,Job{shared:self.shared.clone(),packet})?; for sync in syncs.drain(0..in_sync_count) {if let Some(f)=sync.fence {job.add_dependency(f)?;}} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 79071cbc434a0..7a5587ef6ec78 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -15,39 +15,9 @@ use kernel::page::PAGE_SIZE; use crate::debug::*; use crate::mmu; -use core::sync::atomic::{AtomicBool, Ordering}; type Asid = u8; -/// Invalidate ranges page by page (`mmu::UatQuirks::page_tlbi`): never range operations. -/// -/// The setting is global because the invalidation helpers take no UAT. It assumes one GPU per -/// system: `Uat::new` sets it for that GPU before the GPU's first invalidation, and a second -/// UAT with another policy would change it for both. -static PAGE_TLBI: AtomicBool = AtomicBool::new(false); - -/// Ranges above this many pages are invalidated by ASID in page mode. -const PAGE_TLBI_MAX_PAGES: usize = 2048; - -/// Selects page-by-page range invalidation for the system's GPU ([`PAGE_TLBI`]). -pub(crate) fn set_page_tlbi(enabled: bool) { - PAGE_TLBI.store(enabled, Ordering::Relaxed); -} - -/// Invalidates `len` bytes at `va` one 16 KiB page at a time, or the whole ASID above -/// [`PAGE_TLBI_MAX_PAGES`]. -fn tlbi_pages(asid: Asid, va: usize, len: usize) { - let start = va >> mmu::UAT_PGBIT; - let end = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; - if end - start > PAGE_TLBI_MAX_PAGES { - tlbi_asid(asid); - return; - } - for page in start..end { - tlbi_page(asid, page << mmu::UAT_PGBIT); - } -} - /// Invalidate the entire GPU TLB. #[inline(always)] pub(crate) fn tlbi_all() { @@ -71,7 +41,7 @@ pub(crate) fn tlbi_asid(asid: Asid) { asm!( ".arch armv8.4-a", "tlbi aside1os, {x}", - x = in(reg) (u64::from(asid) << 48) + x = in(reg) ((asid as u64) << 48) ); } } @@ -85,7 +55,7 @@ pub(crate) fn tlbi_page(asid: Asid, va: usize) { return; } - let val: u64 = (u64::from(asid) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); // SAFETY: tlbi is always safe by definition unsafe { asm!( @@ -108,16 +78,12 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { if len == 0 { return; } - if PAGE_TLBI.load(Ordering::Relaxed) { - tlbi_pages(asid, va, len); - return; - } let start_pg = va >> mmu::UAT_PGBIT; let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; const BADDR_MASK: u64 = (1 << 37) - 1; - let mut val: u64 = (u64::from(asid) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); + let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); let pages = end_pg - start_pg; // Guess? It's possible that the page count is in terms of 4K pages @@ -147,7 +113,7 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { let bits = 5 * exp; let mut base = (num + (1 << bits) - 1) >> bits; - val |= u64::from(exp) << 44; + val |= (exp as u64) << 44; while base > 32 { // SAFETY: tlbi is always safe by definition @@ -201,10 +167,6 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { if len == 0 { return; } - if PAGE_TLBI.load(Ordering::Relaxed) { - tlbi_pages(asid, va, len); - return; - } if va == 0 || len >= TLBI_RANGE_MAX { tlbi_asid(asid); @@ -223,7 +185,7 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { }; let num = ((len - 1) >> unit_shift) as u64; - let val = (u64::from(asid) << 48) + let val = ((asid as u64) << 48) | (TLBI_TG_16K << 46) | (scale << 44) | (num << 39) diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs deleted file mode 100644 index 12fc90b23d8eb..0000000000000 --- a/drivers/gpu/drm/asahi/memacct.rs +++ /dev/null @@ -1,549 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU memory attribution, accounting and admission limits. -//! -//! The backing of a GEM object that the GPU holds is pinned shmem: it sits on the -//! unevictable LRU and cannot be swapped out. It is charged to the memory cgroup of -//! the task that faulted it in, but the OOM killer scores a task by the resident set -//! of its mm, and GPU-only pages are mapped into no process. A game holding -//! gigabytes of buffers therefore scores like a tiny process, and so does a -//! compositor; which one an OOM kill hits is left to the other terms of the score. -//! -//! This module closes that gap: -//! -//! - Every user object remembers the mm of the process that created it (an `mmgrab` -//! reference, which keeps only the `mm_struct` itself). While the GPU holds the -//! object's backing, its pages are added to that mm's `MM_SHMEMPAGES` counter. -//! They are shmem pages, so this is where `oom_badness()`, the OOM report's -//! `rss_shmem` column and `RssShmem`/`VmRSS` in `/proc//status` look. Pages -//! are uncharged when idle reclaim or purge releases the backing and when the -//! object is freed; the mm reference is dropped after the last uncharge, so the -//! counter is back to its own value before `__mmdrop()` checks it. -//! - Each client (DRM file) and the device keep totals of created and held bytes, -//! reported in fdinfo (`drm-total-system`, `drm-resident-system`) and in the -//! `gpu_memory` sysfs file next to `agx_stats`. -//! - A new user object is admitted only while its client stays within a per-client -//! limit and the device within a device-wide limit, both fractions of RAM. -//! A refused allocation fails with `ENOMEM`, which userspace reports as -//! out-of-device-memory, instead of pushing the machine into a global OOM. -//! -//! Imported dma-bufs are charged to their exporter and are not counted here. -//! Kernel objects (firmware structures, per-queue buffers and the global heaps) -//! count towards the device total and its limit, but are attributed to no process. - -use core::sync::atomic::{ - AtomicBool, - AtomicU64, - Ordering, // -}; - -use kernel::{ - mm::{ - Mm, - RssCounter, // - }, - page::PAGE_SIZE, - prelude::*, - sync::{aref::ARef, Arc}, -}; - -use crate::module_parameters; - -/// Device-wide GPU memory totals, in bytes. -/// -/// The layout is shared with the C sysfs shim (`struct asahi_gpu_memory`). -#[repr(C)] -pub(crate) struct Ledger { - /// Size of every live user object (created, not imported). - pub(crate) user_committed: AtomicU64, - /// Size of the live user objects whose backing the GPU currently holds. - pub(crate) user_resident: AtomicU64, - /// Size of the live kernel objects whose backing the GPU holds. - pub(crate) kernel_resident: AtomicU64, - /// User object creations refused by a limit. - pub(crate) refused: AtomicU64, - /// Device-wide limit on user committed plus kernel resident bytes (0: none). - pub(crate) limit: AtomicU64, - /// Per-client limit on committed bytes (0: none). - pub(crate) client_limit: AtomicU64, -} - -// Keep the Rust half of the C sysfs layout contract checked as well. -const _: () = { - assert!(core::mem::offset_of!(Ledger, refused) == 24); - assert!(core::mem::offset_of!(Ledger, client_limit) == 40); - assert!(core::mem::size_of::() == 48); -}; - -/// The ledger of the (single) AGX GPU. -pub(crate) static LEDGER: Ledger = Ledger::new(); - -/// The limits that follow from the amount of RAM and two percentages. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) struct Limits { - /// Device-wide limit in bytes (0: none). - pub(crate) device: u64, - /// Per-client limit in bytes (0: none). - pub(crate) client: u64, -} - -fn pct_of(total: u64, pct: u32) -> u64 { - match pct { - 0 => 0, - // Above 100% cannot be held anyway; treat it as "no limit". - p if p >= 100 => 0, - p => total / 100 * u64::from(p) + total % 100 * u64::from(p) / 100, - } -} - -/// Computes the limits for `total_ram` bytes of RAM. -/// -/// A percentage of 0 or 100 and above disables that limit. A per-client limit -/// is never larger than the device limit. -pub(crate) fn limits_for(total_ram: u64, device_pct: u32, client_pct: u32) -> Limits { - let device = pct_of(total_ram, device_pct); - let mut client = pct_of(total_ram, client_pct); - if device != 0 && (client == 0 || client > device) { - client = device; - } - Limits { device, client } -} - -/// Adds `bytes` to `counter` unless the result would exceed `limit` (0: no limit). -/// -/// Returns whether the bytes were added. Concurrent callers cannot overshoot. -fn try_add_within(counter: &AtomicU64, bytes: u64, limit: u64, extra: u64) -> bool { - let mut cur = counter.load(Ordering::Relaxed); - loop { - let Some(new) = cur.checked_add(bytes) else { - return false; - }; - if limit != 0 && new.saturating_add(extra) > limit { - return false; - } - match counter.compare_exchange_weak(cur, new, Ordering::AcqRel, Ordering::Relaxed) { - Ok(_) => return true, - Err(actual) => cur = actual, - } - } -} - -fn sub(counter: &AtomicU64, bytes: u64) { - let prev = counter.fetch_sub(bytes, Ordering::AcqRel); - if prev < bytes { - pr_warn!("GPU memory accounting underflow ({} < {})\n", prev, bytes); - counter.store(0, Ordering::Relaxed); - } -} - -impl Ledger { - pub(crate) const fn new() -> Self { - Self { - user_committed: AtomicU64::new(0), - user_resident: AtomicU64::new(0), - kernel_resident: AtomicU64::new(0), - refused: AtomicU64::new(0), - limit: AtomicU64::new(0), - client_limit: AtomicU64::new(0), - } - } - - /// Sets the limits from the module parameters and the amount of RAM. - pub(crate) fn configure(&self) { - // SAFETY: `struct sysinfo` is plain integers, for which all zeroes is a valid value. - let mut info: kernel::bindings::sysinfo = unsafe { core::mem::zeroed() }; - // SAFETY: `si_meminfo()` only fills in the structure it is given. - unsafe { kernel::bindings::si_meminfo(&mut info) }; - let total = (info.totalram as u64).saturating_mul(u64::from(info.mem_unit)); - let limits = limits_for( - total, - module_parameters::mem_limit_pct.value(), - module_parameters::client_mem_limit_pct.value(), - ); - self.set_limits(limits); - pr_info!( - "GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", - limits.device >> 20, - limits.client >> 20 - ); - } - - /// Admission capacities exported in the optional GET_PARAMS tail. - pub(crate) fn limits(&self) -> Limits { - Limits { - device: self.limit.load(Ordering::Relaxed), - client: self.client_limit.load(Ordering::Relaxed), - } - } - - pub(crate) fn set_limits(&self, limits: Limits) { - self.limit.store(limits.device, Ordering::Relaxed); - self.client_limit.store(limits.client, Ordering::Relaxed); - } - - /// Admits a new user object of `bytes` for `client`. - /// - /// Reserves the bytes against both limits; the returned [`Commit`] gives - /// them back when it is dropped. - pub(crate) fn admit(&'static self, client: &Arc, bytes: u64) -> Result { - let client_limit = self.client_limit.load(Ordering::Relaxed); - if !try_add_within(&client.committed, bytes, client_limit, 0) { - self.refused.fetch_add(1, Ordering::Relaxed); - return Err(ENOMEM); - } - // Kernel objects share the device limit but are never refused. - let kernel = self.kernel_resident.load(Ordering::Relaxed); - if !try_add_within( - &self.user_committed, - bytes, - self.limit.load(Ordering::Relaxed), - kernel, - ) { - sub(&client.committed, bytes); - self.refused.fetch_add(1, Ordering::Relaxed); - return Err(ENOMEM); - } - Ok(Commit { - ledger: self, - client: client.clone(), - bytes, - }) - } -} - -/// Per-client (DRM file) totals, in bytes. -pub(crate) struct Client { - /// Size of the live user objects this client created. - committed: AtomicU64, - /// Size of those whose backing the GPU currently holds. - resident: AtomicU64, -} - -impl Client { - pub(crate) fn new() -> Result> { - Ok(Arc::new( - Self { - committed: AtomicU64::new(0), - resident: AtomicU64::new(0), - }, - GFP_KERNEL, - )?) - } - - pub(crate) fn committed(&self) -> u64 { - self.committed.load(Ordering::Relaxed) - } - - pub(crate) fn resident(&self) -> u64 { - self.resident.load(Ordering::Relaxed) - } -} - -/// Bytes of a user object reserved against the limits. -pub(crate) struct Commit { - ledger: &'static Ledger, - client: Arc, - bytes: u64, -} - -impl Drop for Commit { - fn drop(&mut self) { - sub(&self.client.committed, self.bytes); - sub(&self.ledger.user_committed, self.bytes); - } -} - -enum Kind { - /// An object the DRM core created (a dma-buf import): charged to its exporter. - None, - /// A driver-owned object. - Kernel(&'static Ledger), - /// A user object, attributed to the process that created it. - User { - commit: Commit, - mm: Option>, - }, -} - -/// The accounting state of one GEM object. -pub(crate) struct Charge { - kind: Kind, - bytes: u64, - /// Whether the bytes are currently counted as held by the GPU. - held: AtomicBool, -} - -// SAFETY: `ARef` may be dropped on any thread and the counters are atomic. -unsafe impl Send for Charge {} -// SAFETY: Shared access only touches atomics and the per-cpu RSS counters. -unsafe impl Sync for Charge {} - -impl Charge { - /// An object that is not counted (a dma-buf import). - pub(crate) fn none() -> Self { - Self { - kind: Kind::None, - bytes: 0, - held: AtomicBool::new(false), - } - } - - /// A kernel object of `bytes`. - pub(crate) fn kernel(ledger: &'static Ledger, bytes: u64) -> Self { - Self { - kind: Kind::Kernel(ledger), - bytes, - held: AtomicBool::new(false), - } - } - - /// A user object admitted by `commit`, attributed to `mm` if there is one. - pub(crate) fn user(commit: Commit, mm: Option>) -> Self { - let bytes = commit.bytes; - let mm = if module_parameters::oom_attribution.value() != 0 { - mm - } else { - None - }; - Self { - kind: Kind::User { commit, mm }, - bytes, - held: AtomicBool::new(false), - } - } - - /// A user object attributed to the current process. - pub(crate) fn user_current(commit: Commit) -> Self { - let mm = kernel::current!().mm().map(|mm| ARef::from(&**mm)); - Self::user(commit, mm) - } - - fn pages(&self) -> isize { - isize::try_from(self.bytes / PAGE_SIZE as u64).unwrap_or(isize::MAX) - } - - fn apply(&self, held: bool) { - match &self.kind { - Kind::None => {} - Kind::Kernel(ledger) => { - if held { - ledger.kernel_resident.fetch_add(self.bytes, Ordering::AcqRel); - } else { - sub(&ledger.kernel_resident, self.bytes); - } - } - Kind::User { commit, mm } => { - if held { - commit.client.resident.fetch_add(self.bytes, Ordering::AcqRel); - commit - .ledger - .user_resident - .fetch_add(self.bytes, Ordering::AcqRel); - } else { - sub(&commit.client.resident, self.bytes); - sub(&commit.ledger.user_resident, self.bytes); - } - if let Some(mm) = mm { - let pages = self.pages(); - mm.add_rss_counter(RssCounter::Shmem, if held { pages } else { -pages }); - } - } - } - } - - /// Records that the GPU now holds (or no longer holds) the object's backing. - /// - /// Idempotent: only a change of state is counted. - pub(crate) fn set_held(&self, held: bool) { - if self.held.swap(held, Ordering::AcqRel) != held { - self.apply(held); - } - } -} - -impl Drop for Charge { - fn drop(&mut self) { - // Uncharge before `kind` (and with it the mm reference) is dropped. - self.set_held(false); - } -} - -#[kunit_tests(asahi_memacct)] -mod tests { - use super::*; - - const GIB: u64 = 1 << 30; - const MIB: u64 = 1 << 20; - - #[test] - fn default_limits_follow_ram() { - // The module parameter defaults. - let l = limits_for(8 * GIB, 67, 50); - assert_eq!(l.device, 8 * GIB * 67 / 100); - assert_eq!(l.client, 4 * GIB); - } - - #[test] - fn disabled_and_clamped_limits() { - assert_eq!(limits_for(8 * GIB, 0, 0), Limits { device: 0, client: 0 }); - assert_eq!(limits_for(8 * GIB, 100, 0), Limits { device: 0, client: 0 }); - // A client limit above the device limit is clamped to it. - let l = limits_for(8 * GIB, 25, 50); - assert_eq!(l.client, l.device); - // No client limit with a device limit means the device limit. - let l = limits_for(8 * GIB, 50, 0); - assert_eq!(l.client, 4 * GIB); - // No device limit leaves the client limit alone. - let l = limits_for(8 * GIB, 0, 25); - assert_eq!(l, Limits { device: 0, client: 2 * GIB }); - } - - #[test] - fn percentages_do_not_overflow() { - let l = limits_for(u64::MAX, 99, 98); - assert!(l.device > l.client); - assert!(l.device <= u64::MAX / 100 * 99 + 99); - } - - #[test] - fn try_add_respects_limit_and_extra() { - let c = AtomicU64::new(0); - assert!(try_add_within(&c, 6, 10, 0)); - assert!(!try_add_within(&c, 5, 10, 0)); - assert!(try_add_within(&c, 4, 10, 0)); - assert_eq!(c.load(Ordering::Relaxed), 10); - let d = AtomicU64::new(0); - assert!(!try_add_within(&d, 6, 10, 5)); - assert!(try_add_within(&d, 5, 10, 5)); - let e = AtomicU64::new(u64::MAX - 1); - assert!(!try_add_within(&e, 2, 0, 0)); - assert!(try_add_within(&e, 1, 0, 0)); - } - - static ADMIT: Ledger = Ledger::new(); - - #[test] - fn admission_reserves_and_releases() -> Result { - ADMIT.set_limits(Limits { device: 100 * MIB, client: 60 * MIB }); - let a = Client::new()?; - let b = Client::new()?; - let c = Client::new()?; - let a1 = ADMIT.admit(&a, 40 * MIB)?; - // The client limit refuses a's second 40 MiB; nothing stays reserved. - assert!(ADMIT.admit(&a, 40 * MIB).is_err()); - assert_eq!(a.committed(), 40 * MIB); - assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 40 * MIB); - let b1 = ADMIT.admit(&b, 50 * MIB)?; - // The device limit refuses c's 20 MiB and rolls back c's reservation. - assert!(ADMIT.admit(&c, 20 * MIB).is_err()); - assert_eq!(c.committed(), 0); - assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 90 * MIB); - assert_eq!(ADMIT.refused.load(Ordering::Relaxed), 2); - drop(a1); - assert_eq!(a.committed(), 0); - let c1 = ADMIT.admit(&c, 20 * MIB)?; - drop((b1, c1)); - assert_eq!(b.committed(), 0); - assert_eq!(c.committed(), 0); - assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 0); - Ok(()) - } - - static KERNEL: Ledger = Ledger::new(); - - #[test] - fn kernel_objects_count_against_the_device_limit() -> Result { - KERNEL.set_limits(Limits { device: 64 * MIB, client: 64 * MIB }); - let k = Charge::kernel(&KERNEL, 48 * MIB); - k.set_held(true); - let a = Client::new()?; - assert!(KERNEL.admit(&a, 32 * MIB).is_err()); - let c = KERNEL.admit(&a, 16 * MIB)?; - drop(k); - assert_eq!(KERNEL.kernel_resident.load(Ordering::Relaxed), 0); - drop(c); - Ok(()) - } - - static HELD: Ledger = Ledger::new(); - - #[test] - fn held_transitions_are_idempotent() -> Result { - HELD.set_limits(Limits { device: 0, client: 0 }); - let a = Client::new()?; - let charge = Charge::user(HELD.admit(&a, 8 * MIB)?, None); - assert!(!charge.held.load(Ordering::Relaxed)); - charge.set_held(true); - charge.set_held(true); - assert_eq!(a.resident(), 8 * MIB); - assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 8 * MIB); - charge.set_held(false); - charge.set_held(false); - assert_eq!(a.resident(), 0); - charge.set_held(true); - // Dropping a held object uncharges it and returns its commit. - drop(charge); - assert_eq!(a.resident(), 0); - assert_eq!(a.committed(), 0); - assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 0); - assert_eq!(HELD.user_committed.load(Ordering::Relaxed), 0); - Ok(()) - } - - #[test] - fn imports_are_not_counted() { - let charge = Charge::none(); - charge.set_held(true); - assert!(charge.held.load(Ordering::Relaxed)); - drop(charge); - } - - static MM: Ledger = Ledger::new(); - - // `mm_alloc()` is only exported to modules in the KUnit namespace. - #[cfg(not(MODULE))] - #[test] - fn held_pages_are_charged_to_the_creating_mm() -> Result { - MM.set_limits(Limits { device: 0, client: 0 }); - // SAFETY: `mm_alloc()` returns a new mm with one user and one count, or NULL. - let raw = unsafe { kernel::bindings::mm_alloc() }; - if raw.is_null() { - return Err(ENOMEM); - } - // SAFETY: `raw` is a live mm until the `mmput()` below. - let mm = ARef::::from(unsafe { Mm::from_raw(raw) }); - let a = Client::new()?; - let charge = Charge::user(MM.admit(&a, 4 * PAGE_SIZE as u64)?, Some(mm.clone())); - assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); - charge.set_held(true); - assert_eq!(mm.rss_counter(RssCounter::Shmem), 4); - assert_eq!(mm.rss_counter(RssCounter::Anon), 0); - charge.set_held(false); - assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); - charge.set_held(true); - // Freeing a held object uncharges it before dropping its mm reference. - drop(charge); - assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); - drop(mm); - // SAFETY: Drops the user reference `mm_alloc()` returned. `__mmdrop()` then - // checks that every RSS counter is back to zero. - unsafe { kernel::bindings::mmput(raw) }; - Ok(()) - } -} - -#[kernel::macros::kunit_tests(asahi_gem_memory_limits)] -mod exported_limits_tests { - use super::*; - - #[test] - fn exported_capacities_match_admission() { - let ledger = Ledger::new(); - assert_eq!(ledger.limits(), Limits { device: 0, client: 0 }); - let limits = limits_for(8 << 30, 67, 50); - ledger.set_limits(limits); - assert_eq!(ledger.limits(), limits); - assert!(limits.client < limits.device); - assert_eq!(limits.client, 4 << 30); - } -} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 437685bfad1de..2bf4a6823024d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -15,7 +15,7 @@ //! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; -use core::mem::{size_of, ManuallyDrop}; +use core::mem::size_of; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ @@ -29,6 +29,7 @@ use core::sync::atomic::{ use kernel::{ addr::{ + DmaAddr, PhysicalAddr, // }, c_str, @@ -108,19 +109,20 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; +use pin_init; + const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; +mod shared; mod lifetime; -mod ppl; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod m3; mod residency; -mod shared; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; @@ -154,81 +156,23 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// One or two coprocessor instances that never touch the handoff region (G17). + /// Two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at - /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. - /// TTBAT context 1 starts as an alias of the kernel context. + /// primary instance uses the `pagetables` root, the secondary one the root at + /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel + /// context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: Option, + secondary_root_offset: u64, }, } -/// UAT policy of a SoC where it differs from the A18 Pro's. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) struct UatQuirks { - /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and - /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of - /// `struct page`. - /// - /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the - /// regions lose `no-map` and this path goes away. - pub(crate) reserved_tables: bool, - // The following settings keep a SoC on the only policy it was tested with. Each one is - // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. - /// Write back every page-table and context-table change: the GPU table walker is not - /// assumed to snoop the CPU caches. - pub(crate) clean_tables: bool, - /// Invalidate ranges one page at a time, never with range TLBI operations. - pub(crate) page_tlbi: bool, - /// Take the handoff lock around context-table edits. Where the firmware leaves its half of - /// the handoff page alone, its lock bytes are not initialized and are never taken. - pub(crate) handoff_lock: bool, - /// Handoff "current slot" written at initialization. - pub(crate) handoff_slot: u32, - /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in - /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. - pub(crate) native_gpu_permissions: bool, -} - -impl UatQuirks { - /// The A18 Pro and earlier policy. - pub(crate) const DEFAULT: UatQuirks = UatQuirks { - reserved_tables: false, - clean_tables: false, - page_tlbi: false, - handoff_lock: true, - handoff_slot: 0, - native_gpu_permissions: false, - }; -} - const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; -/// M4's nonzero wire incarnation of a host-owned root installation. -const fn binding_wire_generation(incarnation: u64) -> u8 { - if incarnation == 0 { - 0 - } else { - (1 + (incarnation - 1) % 255) as u8 - } -} - -/// Upper word of an application TTBAT slot. M4 exposes firmware upper mappings only in -/// context 0; the existing G15 handoff shares them with user contexts. -const fn application_ttb1(root: u64, asid: usize, map_kernel_to_user: bool) -> u64 { - if map_kernel_to_user { - root | TTBR_VALID | (asid as u64) << TTBR_ASID_SHIFT - } else { - 0 - } -} - /// UAT address space geometry of a given SoC. #[derive(Debug, Copy, Clone)] pub(crate) struct UatGeometry { @@ -414,28 +358,11 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] -pub(crate) struct VmReadGuard(VmExecGuard); - -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] -impl VmReadGuard { - #[cfg(CONFIG_DRM_ASAHI_G15)] - pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { - Vm::covers_range_locked(&mut self.0, address, size, read, write) - } - - pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { - Vm::read_user_locked(&mut self.0, va, buf) - } -} - /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, binding: Option>, bind_token: Option, - /// Nonzero M4 wire incarnation of the root currently owning this slot. - generation: u8, ttb: u64, } @@ -448,20 +375,7 @@ struct VmBoData { /// linking the GPUVA. Remap pieces keep their original sequence. newest_bind: AtomicU64, #[pin] - segments: Mutex>, -} - -/// Copied DMA addresses and the lease covering every alias of this VM/BO association. -/// The lease is dropped only after all aliases have been unmapped and invalidated. -struct SgSegments { - segments: KVVec<(DmaAddress, ResourceSize)>, - _lease: kernel::drm::gem::shmem::SGLease, -} - -impl SgSegments { - fn iter(&self) -> core::slice::Iter<'_, (DmaAddress, ResourceSize)> { - self.segments.iter() - } + segments: Mutex>>, } /// Preallocated resources needed to execute a VM operation. @@ -580,16 +494,22 @@ impl gpuvm::DriverGpuVm for VmInner { let one_page = op.flags().contains(gpuvm::GpuVaFlag::REPEAT); - if ctx.dev.is_none() { + let Some(dev) = ctx.dev else { + dev_err!(self.dev.as_ref(), "StepContext missed Bound device!"); return Err(ENODEV); - } - // Bind preparation retained the table's backing before taking the VM reservation. - // Never acquire the object reservation from inside the GPUVM map callback. - let segments = op.vm_bo().data().segments.lock(); - let sgt = segments.as_ref().ok_or(EFAULT)?; - for &(dma_address, dma_len) in sgt.iter() { - let mut addr = dma_address as usize; - let mut len = dma_len as usize; + }; + + let sgt = op.obj().sg_table(dev).inspect_err(|e| { + dev_err!( + self.dev.as_ref(), + "sm_step_map() Failed to get SG table: {:?}", + e + ); + })?; + for sgt_entry in sgt.iter() { + // TODO: proper DMA address/length handling + let mut addr = sgt_entry.dma_address() as usize; + let mut len: usize = sgt_entry.dma_len() as usize; if left == 0 { break; @@ -636,7 +556,6 @@ impl gpuvm::DriverGpuVm for VmInner { iova += len as u64; } - drop(segments); let gpuva = match ctx.preallocated_gpuva() { Ok(gpuva) => gpuva, Err(error) => { @@ -722,8 +641,7 @@ impl gpuvm::DriverGpuVm for VmInner { let _mutation = self.mapping_mutation(); if orig_invalidated { - self.page_table - .discard_partial_map(unmap_start..unmap_end)?; + self.page_table.discard_partial_map(unmap_start..unmap_end)?; } else { self.page_table.unmap_pages(unmap_start..unmap_end)?; } @@ -766,12 +684,7 @@ impl gpuvm::DriverGpuVm for VmInner { impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!( - self.dev, - "MMU: unmap: {:#x}:{:#x}\n", - va.addr(), - va.length() - ); + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); let _mutation = self.mapping_mutation(); if va.flags().contains(GpuVaFlag::INVALIDATED) { @@ -806,7 +719,14 @@ impl VmInner { // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); - self.finish_failed_map(addr, size as usize); + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + mem::sync(); + self.tlbi_range(asid as u8, addr, size as usize); + mem::sync(); + } + } + self.tlbi_contexts_naming_root(addr, size as usize); if let Err(error) = result { dev_err!( self.dev.as_ref(), @@ -816,19 +736,6 @@ impl VmInner { } } - /// Complete discarded stores and invalidate both the legacy bound slot and - /// every M4/G17 context naming this root. Call only after releasing PPL. - fn finish_failed_map(&self, addr: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - mem::sync(); - self.tlbi_range(asid as u8, addr, size); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size); - } - fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { @@ -883,19 +790,15 @@ impl VmInner { /// /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Adopted M4 roots also remain in the TTBAT after a bind - /// lease is dropped, until the slot is reassigned or the root is destroyed. Include those - /// contexts before allowing an unmapped object's backing to be released. + /// change must reach all of them. Other firmware only uses the bind slot, which the callers + /// already invalidate. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff && !self.uat_inner.m4_adopted { + if self.uat_inner.firmware == UatFirmware::Handoff { mem::sync(); return; } - let contexts = self - .uat_inner - .lock() - .contexts_naming_root(self.ttb(), self.uat_inner.m4_adopted); + let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); self.tlbi_context_mask(iova, size, contexts); } @@ -923,35 +826,10 @@ impl VmInner { .checked_add(node.mapped_size as u64) .ok_or(EOVERFLOW)?; let _mutation = self.mapping_mutation(); - // Allocate detached RAM tables before excluding the firmware. The shadow L2 is not - // the reserved shared-L2; only final leaf stores below already linked L3s are live. self.page_table.prepare_map(node.start()..end)?; - let uat_inner = self.uat_inner.clone(); - let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; let result = self.map_node_prepared(node, prot); - let discard = if result.is_err() { - // Rollback still changes live leaves and must remain under the PPL lease. - self.page_table.discard_partial_map(node.start()..end) - } else { - Ok(()) - }; - // The M4 handoff guard owns UatShared's mutex. Invalidation follows its release. - drop(handoff); if result.is_err() { - self.finish_failed_map(node.start(), node.mapped_size); - } else { - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); - } - if let Err(error) = discard { - if self.uat_inner.m4_adopted { - // The caller retains its node and backing when this latch is set. - self.uat_inner.failed.store(true, Ordering::Release); - } - dev_err!( - self.dev.as_ref(), - "Could not discard failed kernel mapping: {:?}\n", - error - ); + self.rollback_failed_map(node.start(), node.mapped_size as u64); } result } @@ -959,14 +837,7 @@ impl VmInner { /// Install only into the empty range reserved by `map_node`. fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node - .bo - .as_ref() - .ok_or(EINVAL)? - .get()? - .data() - .segments - .lock(); + let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -1024,7 +895,8 @@ impl VmInner { } } - // The caller invalidates negative translations after releasing its PPL lease. + // A context naming this root may be live and hold a negative translation for the range. + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); Ok(()) } } @@ -1032,7 +904,6 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { - device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, @@ -1103,7 +974,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff_lock(); + inner.handoff().lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -1111,9 +982,8 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); - inner.publish_slot(id); } - inner.handoff_unlock(); + inner.handoff().unlock(); owned || (low == 0 && high == 0) }; if release { @@ -1154,12 +1024,6 @@ impl VmBind { pub(crate) fn slot(&self) -> u32 { self.1 } - - /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn generation(&self) -> u8 { - self.0.binding.lock().generation - } } impl Drop for VmBind { @@ -1200,7 +1064,10 @@ struct MappingBo { owner: ARef>, } impl MappingBo { - fn new(bo: ARef>, owner: ARef>) -> Self { + fn new( + bo: ARef>, + owner: ARef>, + ) -> Self { Self { bo: Some(bo), owner, @@ -1243,90 +1110,29 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's -/// reservation lock, so it must not be dropped where fence producers run. -pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); - -impl TimestampVmap { - pub(crate) fn new(gem: &gem::Object) -> Result { - Ok(Self(gem.owned_vmap::<0>()?)) - } - - fn size(&self) -> usize { - use kernel::drm::gem::BaseObject; - self.0.owner().size() - } - - /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete - /// atomic store; byte stores support unaligned destinations without creating mutable - /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused - /// when the destination is prepared, and ignored here. - pub(crate) fn write(&self, offset: usize, value: u64) { - if offset - .checked_add(size_of::()) - .is_none_or(|end| end > self.size()) - { - return; - } - // Completion runs in task context. Bindings of the same BO share this lock, - // so aligned u64 and overlapping unaligned byte stores never race in Rust. - let _guard = self.0.owner().timestamp_lock.lock(); - let base = self.0.addr.cast::(); - // SAFETY: The complete extent was checked above, inside this pinned vmap. - let pointer = unsafe { base.add(offset) }; - if pointer as usize % core::mem::align_of::() == 0 { - // SAFETY: All eight bytes were checked above; the vmap pins the backing. - // A naturally aligned query result becomes available in one complete store. - let destination = unsafe { &*pointer.cast::() }; - destination.store(value.to_le(), Ordering::Relaxed); - } else { - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: The complete eight-byte extent was checked above. The owned - // vmap pins its backing; byte atomics permit unaligned and overlapping - // timestamp destinations without mutable slice aliases. - let destination = unsafe { &*base.add(offset + index).cast::() }; - destination.store(byte, Ordering::Relaxed); - } - } - // A CPU-retired query can be consumed by a later GPU command. WRITEBACK - // objects use a cacheable vmap, while GPU mappings need not snoop that cache. - // Publish the bytes before signalling the completion fence or queue frontier. - if self.0.owner().is_writeback() { - crate::pgtable::clean_poc(pointer, size_of::()); - } - crate::pgtable::poc_barrier(); - } -} - -/// Prepared CPU timestamp access with its own vmap; destruction happens after completion -/// signalling. -#[cfg(CONFIG_DRM_ASAHI_G16)] +/// Prepared CPU timestamp access; destruction happens after completion signalling. pub(crate) struct TimestampMapping { - mapping: TimestampVmap, + mapping: kernel::drm::gem::shmem::VMapOwned, offset: usize, } -#[cfg(CONFIG_DRM_ASAHI_G16)] impl TimestampMapping { - /// Prepare one bounded CPU timestamp destination on retained GEM backing. - pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { - use kernel::drm::gem::BaseObject; - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { - return Err(ERANGE); - } - Ok(Self { - mapping: TimestampVmap::new(gem)?, - offset, - }) - } - pub(crate) fn write(&self, value: u64) { - self.mapping.write(self.offset, value); + let base = self.mapping.addr.cast::(); + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: timestamp_mapping checked the complete eight-byte extent. + // The owned vmap pins its backing; byte atomics permit unaligned and + // overlapping timestamp destinations without mutable slice aliases. + let destination = unsafe { + &*base.add(self.offset + index).cast::() + }; + destination.store(byte, Ordering::Relaxed); + } } } /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(ManuallyDrop>); +pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); impl KernelMapping { /// Returns the IOVA base of this mapping @@ -1350,22 +1156,14 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - self.0.owner.lock_inner().page_table.translate_iova(address) + lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) } - /// Writes a userspace timestamp through the retained GEM mapping. Aligned results - /// use a complete atomic store; byte stores support unaligned destinations without - /// creating mutable aliases. An unaligned complete timestamp is not atomic. + /// Writes a userspace timestamp through the retained GEM mapping. Byte + /// stores support unaligned destinations and concurrent completion writes + /// without creating mutable aliases. The complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. - #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { - let gem = self.0._gem.as_ref().ok_or(EINVAL)?; - TimestampMapping::new(gem, self.timestamp_offset(offset)?) - } - - /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete - /// extent lies inside both the mapping and the object. - pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1376,12 +1174,10 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - Ok(start) - } - - /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. - pub(crate) fn timestamp_vmap(&self) -> Result { - TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) + Ok(TimestampMapping { + mapping: gem.owned_vmap::<0>()?, + offset: start, + }) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1493,13 +1289,13 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff_lock(); + uat_inner.handoff().lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff_unlock(); + uat_inner.handoff().unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1589,19 +1385,6 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { - // An ASC-cached M4 mapping has no qualified cache-flush teardown. Even an unexpected - // failed construction must retain its complete node/backing instead of invoking the - // unrelated G13/G14 FwCtl protocol. - if self.0.uat_inner.m4_adopted && self.0.prot.is_cached_noncoherent() { - self.0.uat_inner.failed.store(true, Ordering::Release); - pr_err!( - "MMU: retaining cached M4 mapping {:#x}+{:#x} until reboot\n", - self.iova(), - self.size() - ); - return; - } - // This is the main unmap function for UAT mappings. // The sequence of operations here is finicky, due to the interaction // between cached GFX ASC mappings and the page tables. These mappings @@ -1644,27 +1427,8 @@ impl Drop for KernelMapping { self.size() ); - let upper_handoff = match self.0.uat_inner.lock_m4_upper(owner.is_kernel) { - Ok(guard) => guard, - Err(error) => { - dev_err!( - owner.dev.as_ref(), - "MMU: retaining M4 mapping after PPL failure: {:?}\n", - error - ); - return; - } - }; let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { - if self.0.uat_inner.m4_adopted { - self.0.uat_inner.failed.store(true, Ordering::Release); - dev_err!( - owner.dev.as_ref(), - "MMU: retaining M4 mapping after partial unmap failure\n" - ); - return; - } dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -1673,8 +1437,6 @@ impl Drop for KernelMapping { ); } - drop(upper_handoff); - if let Some(asid) = owner.slot() { mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); @@ -1691,11 +1453,6 @@ impl Drop for KernelMapping { // The range returns to the VA allocator once this mapping is gone, so no context that // names this root may keep a stale translation for it. owner.tlbi_contexts_naming_root(self.iova(), self.size()); - drop(_mutation); - drop(owner); - // SAFETY: Every normal path reaches this exactly once, after all GPUVM locks have - // been released. Uncertain M4 paths return earlier, retaining the node and its pins. - unsafe { ManuallyDrop::drop(&mut self.0) }; } } @@ -1704,73 +1461,32 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], - /// Firmware-observed incarnations of application roots installed by the VM slot allocator. - binding_generations: [u64; UAT_NUM_CTX], - binding_published: [bool; UAT_NUM_CTX], - /// Expected lower words of user roots owned by the VM slot allocator. Replaced only by - /// successful host publications, never learned by accepting observed hardware words. - owned_user_roots: [u64; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, - quirks: UatQuirks, } impl UatShared { - /// Takes the handoff lock, where this SoC uses it. - fn handoff_lock(&self) { - if self.quirks.handoff_lock { - self.handoff().lock(); - } - } - - /// Releases the handoff lock taken by [`Self::handoff_lock`]. - fn handoff_unlock(&self) { - if self.quirks.handoff_lock { - self.handoff().unlock(); - } - } - - /// Writes back context-table slot `idx` after it changed, where this SoC needs it. - fn publish_slot(&self, idx: usize) { - if self.quirks.clean_tables { - let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); - pgtable::clean_poc(slot, size_of::()); - pgtable::poc_barrier(); - } - } - /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() + unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { - self.ttbs_rgn - .map - .ptr() - .cast::<[SlotTTBS; UAT_NUM_CTX]>() - .as_ref() - } - .unwrap() + unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64, adopted: bool) -> u64 { + fn contexts_naming_root(&self, ttb: u64) -> u64 { self.ttbs() .iter() .enumerate() - .filter(|(ctx, slot)| { - let expected = tagged_root(ttb, *ctx); - let _ = adopted; // Context 0 is naturally untagged; user contexts carry their ASID. - slot.ttb0.load(Ordering::Acquire) == expected - }) + .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } } @@ -1787,138 +1503,20 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, - /// The stopped J713 handoff: bounded full-barrier PPL and upper words zero in users. - m4_adopted: bool, - /// An uncertain mapping/root is retained; future M4 allocation and publication fail. - failed: AtomicBool, - /// Clean page tables to the point of coherency for a walker that does not snoop the CPU - /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: bool, - /// This SoC's UAT policy. - quirks: UatQuirks, - /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). - reserved: Option>, #[pin] shared: Mutex, #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } -/// A transport owns the sole host-side PPL mutex before touching AP interest. -struct M4Transport<'a> { - shared: Guard<'a, UatShared, MutexBackend>, - start: Instant, - failed: &'a AtomicBool, -} - -impl ppl::Memory for M4Transport<'_> { - fn firmware_interested(&self) -> bool { - self.shared.handoff().lock_fw.load(Ordering::Relaxed) != 0 - } - fn firmware_turn(&self) -> bool { - self.shared.handoff().turn.load(Ordering::Relaxed) != 0 - } - fn ap_interest(&self, interested: bool) { - self.shared - .handoff() - .lock_ap - .store(u8::from(interested), Ordering::Relaxed); - } - fn give_firmware_turn(&self) { - self.shared.handoff().turn.store(1, Ordering::Relaxed); - } - fn barrier(&self) { - mem::sync(); - } - fn expired(&self) -> bool { - self.start.elapsed() > Delta::from_millis(100) - } - fn relax(&self) { - core::hint::spin_loop(); - } - fn fail(&self) { - self.failed.store(true, Ordering::Release); - } -} - -/// Every AP firmware transaction owns the same mutex, including G16 Window operations. -/// M4 drops its protocol guard before its transport releases the host mutex. The legacy -/// handoff path keeps the preexisting protocol and lock ordering for other generations. -#[must_use = "retain the guard until protected table mutations have completed"] -enum HandoffGuard<'a> { - M4(ppl::Guard>), - Legacy(Guard<'a, UatShared, MutexBackend>), -} - -impl HandoffGuard<'_> { - fn shared_mut(&mut self) -> &mut UatShared { - match self { - Self::M4(guard) => &mut guard.memory_mut().shared, - Self::Legacy(shared) => shared, - } - } -} - -impl Drop for HandoffGuard<'_> { - fn drop(&mut self) { - if let Self::Legacy(shared) = self { - shared.handoff_unlock(); - } - } -} - -/// Opaque host-and-firmware lease for G16 window transactions. No caller may -/// independently manipulate the AP-interest or turn words. -#[cfg(CONFIG_DRM_ASAHI_G16)] -#[must_use = "retain the guard throughout the reserved-table window transaction"] -pub(crate) struct M4HandoffGuard<'a> { - _transaction: HandoffGuard<'a>, -} - impl UatInner { /// Take the lock on the shared data and return the guard. fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } - fn check_m4(&self) -> Result { - if self.m4_adopted && self.failed.load(Ordering::Acquire) { - Err(EIO) - } else { - Ok(()) - } - } - - /// Serialize AP callers before taking the firmware exclusion. Timeouts withdraw - /// AP interest while still holding the mutex and latch M4 failure before releasing it. - fn lock_handoff(&self) -> Result> { - self.check_m4()?; - let shared = self.lock(); - self.check_m4()?; - if !self.m4_adopted { - shared.handoff_lock(); - return Ok(HandoffGuard::Legacy(shared)); - } - let transport = M4Transport { - shared, - start: Instant::::now(), - failed: &self.failed, - }; - match ppl::Guard::acquire(transport) { - Ok(guard) => Ok(HandoffGuard::M4(guard)), - Err(()) => Err(ETIMEDOUT), - } - } - - /// No leaf is changed on a failed acquisition. Uncertain mappings retain their backing. - fn lock_m4_upper(&self, upper: bool) -> Result>> { - if upper && self.m4_adopted { - self.lock_handoff().map(Some) - } else { - Ok(None) - } - } - /// Take a lock on a handoff flush slot and return the guard. fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() @@ -1946,9 +1544,7 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 { - core::hint::spin_loop() - } + while self.turn.load(Ordering::Relaxed) != 0 {} self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -1973,9 +1569,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware, slot: u32) -> Result { + fn init(&self, firmware: UatFirmware) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(slot, Ordering::Relaxed); + self.cur_slot.store(0, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -2055,39 +1651,19 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { +fn prefetch_sgt_segments( + dev: &device::Device, + gem: &gem::Object, +) -> Result> { let sgt = crate::gem::held_sg_table(gem, dev)?; + // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(SgSegments { - segments, - _lease: sgt.into_lease(), - }) -} - -/// Reuse cached segments without allocating another copy for each bind. The bind lease was -/// acquired before BO locking: its getter synchronized with any earlier reclaim under resv, -/// and its extra count excludes new eviction until preparation/commit finishes. -fn prepare_sgt_segments( - dev: &device::Device, - gem: &gem::Object, - bo: &gpuvm::GpuVmBo, - _bind_lease: &kernel::drm::gem::shmem::SGLease, -) -> Result { - let needs_segments = bo.data().segments.lock().is_none(); - if needs_segments { - // Never enter the object reservation while holding the BO mutex. - let prefetched = prefetch_sgt_segments(dev, gem)?; - let mut segments = bo.data().segments.lock(); - if segments.is_none() { - segments.replace(prefetched); - } - } - Ok(()) + Ok(segments) } impl Vm { @@ -2110,12 +1686,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, - uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? - .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; + + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -2141,7 +1717,6 @@ impl Vm { VmBinding { binding: None, bind_token: None, - generation: 0, active_users: 0, ttb: page_table.ttb(), }, @@ -2178,9 +1753,7 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); - let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { - device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, @@ -2198,10 +1771,7 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init( - crate::g17::status::VmStatus::new(), - GFP_KERNEL, - )?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); @@ -2212,18 +1782,15 @@ impl Vm { /// Attach a status and job lifetime tracking before sharing a newly created user VM with /// queues, without the G17 canonical context views, shared-object footprints and idle /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn with_tracked_status(mut self) -> Result { - self.status = Some(Arc::pin_init( - crate::g17::status::VmStatus::new(), - GFP_KERNEL, - )?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); Ok(self) } - /// The identifier this VM was created with (G15 keeps per-VM state keyed by it; G17 timeout - /// attribution compares and logs VMs by it). + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn id(&self) -> u64 { self.id } @@ -2235,18 +1802,9 @@ impl Vm { /// and invalidated to the point of coherency first (never only invalidated: CPU-written lines /// may be dirty), so the copy sees what the GPU or a write-combined CPU view stored. The /// mappings cannot change during the copy (GPUVM exec lock). - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - self.read_lock()?.read_user(va, buf) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - pub(crate) fn read_lock(&self) -> Result { - Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { + let mut inner = lock_vm(&self.gpuvm, None, false)?; let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -2316,7 +1874,9 @@ impl Vm { } pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return false; + }; Self::covers_range_locked(&mut inner, address, size, read, write) } @@ -2331,22 +1891,26 @@ impl Vm { if stride <= block_size { return false; } - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return false; + }; addresses.iter().all(|&address| { let Some(guard) = address.checked_add(block_size) else { return false; }; Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked(&mut inner, guard, stride - block_size, false, false) + && !Self::covers_range_locked( + &mut inner, guard, stride - block_size, false, false, + ) }) } - /// Checks all exact request tuples under one hold of the VM reservation. Access bit0 + /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 /// requests reading and bit1 writing. The epoch names this same snapshot. pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { - // Every mapping change holds the VM reservation, so readers need only that one. The full - // exec lock would also lock every external BO, making each submit O(external BOs). - let mut inner = self.gpuvm.lock_inner(); + let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { + return (false, None); + }; let epoch = inner.epoch; let covered = ranges.iter().all(|&(address, size, access)| { Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) @@ -2398,49 +1962,6 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, - ) -> Result { - self.map_in_range_with_mode( - gem, - object_range, - alignment, - range, - prot, - guard_size, - mm::InsertMode::Best, - ) - } - - /// Lowest-address placement for a monotonic search across several VMs. - pub(crate) fn map_in_range_lowest( - &self, - gem: &gem::Object, - object_range: Range, - alignment: u64, - range: Range, - prot: Prot, - guard_size: usize, - ) -> Result { - self.map_in_range_with_mode( - gem, - object_range, - alignment, - range, - prot, - guard_size, - mm::InsertMode::Low, - ) - } - - #[allow(clippy::too_many_arguments)] - fn map_in_range_with_mode( - &self, - gem: &gem::Object, - object_range: Range, - alignment: u64, - range: Range, - prot: Prot, - guard_size: usize, - mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -2455,30 +1976,32 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // End the bound-device lease before entering firmware/GPUVM waits. - let (_bind_lease, vm_bo) = { - let binding = self.device_binding.enter()?; - let dev = binding.device(); - let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let bo = self.gpuvm.obtain( - gem, - pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }), - )?; - prepare_sgt_segments(dev, gem, &bo, &lease)?; - (lease, bo) - }; + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } + let vm_bo = self + .gpuvm + .obtain(gem, pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }))?; let size = object_range.range(); - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)? + self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM @@ -2499,7 +2022,7 @@ impl Vm { 0, range.start, range.end, - mode, + mm::InsertMode::Best, ); let mut node = match result { Ok(node) => node, @@ -2517,16 +2040,9 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. - let retain = ret.is_err() - && inner.uat_inner.m4_adopted - && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Map a GEM object into this Vm at a specific address. @@ -2544,32 +2060,36 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // End the bound-device lease before entering firmware/GPUVM waits. - let (_bind_lease, vm_bo) = { - let binding = self.device_binding.enter()?; - let dev = binding.device(); - let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let bo = self.gpuvm.obtain( - gem, - pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }), - )?; - prepare_sgt_segments(dev, gem, &bo, &lease)?; - (lease, bo) - }; + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } + let vm_bo = self + .gpuvm + .obtain(&gem, pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }))?; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { self.gpuvm - .exec_lock_vm(/*Some(&gem)*/ None, false) + .exec_lock(/*Some(&gem)*/ None, false) .inspect_err(|_| { dev_err!(self.dev, "map_at(): Failed to exec_lock gpuvm\n"); })? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // Keep reservation-locking destructors out of the allocator payload @@ -2604,16 +2124,9 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. - let retain = ret.is_err() - && inner.uat_inner.m4_adopted - && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Map a range of a GEM object into this Vm using GPUVM. @@ -2631,16 +2144,17 @@ impl Vm { let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev: &device::Device = device.as_ref().as_ref(); - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + // TODO: hack, populate the GEM object sg table first + { + let dev = unsafe { self.dev.as_bound() }; + let _sgt = crate::gem::held_sg_table(gem, dev)?; + } let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let mut resources = VmOpResources { preallocated_gpuvas: [ @@ -2653,13 +2167,19 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)? + self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo_guard = vm_bo.data().segments.lock(); + if vm_bo_guard.is_none() { + let dev = unsafe { self.dev.as_bound() }; + let segments = prefetch_sgt_segments(dev, gem)?; + vm_bo_guard.replace(segments); + } + core::mem::drop(vm_bo_guard); + // Preallocate the page tables, to fail early if we ENOMEM inner.page_table.alloc_pages(addr..(addr + size))?; @@ -2702,22 +2222,6 @@ impl Vm { vm_exec.unique_gpuvm.sm_map(req) } - /// Spend the binding generation only for an incarnation exposed to firmware. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn mark_binding_published(&self) -> Result { - let uat = self.gpuvm.lock_inner().uat_inner.clone(); - let binding = self.binding.lock(); - let slot = binding.binding.as_ref().ok_or(EIO)?.slot() as usize + UAT_USER_CTX_START; - let mut shared = uat.lock(); - if shared.owned_user_roots[slot] != tagged_root(binding.ttb, slot) - || shared.ttbs()[slot].ttb0.load(Ordering::Acquire) != shared.owned_user_roots[slot] - { - return Err(EIO); - } - shared.binding_published[slot] = true; - Ok(()) - } - /// Add a direct MMIO mapping to this Vm at a free address. pub(crate) fn map_io( &self, @@ -2727,9 +2231,7 @@ impl Vm { size: usize, prot: Prot, ) -> Result { - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. - let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( @@ -2763,35 +2265,22 @@ impl Vm { let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; - let transaction_uat = inner.uat_inner.clone(); - let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); - let ret = inner - .page_table - .map_pages(iova..end, phys as PhysicalAddr, prot, false); - let discard = if ret.is_err() { - inner.page_table.discard_partial_map(iova..end) - } else { - Ok(()) - }; - drop(handoff); + let ret = inner.page_table.map_pages( + iova..end, + phys as PhysicalAddr, + prot, + false, + ); if ret.is_err() { - inner.finish_failed_map(iova, size); - } else { - inner.tlbi_contexts_naming_root(iova, size); - } - let retain = discard.is_err() && inner.uat_inner.m4_adopted; - if retain { - inner.uat_inner.failed.store(true, Ordering::Release); + inner.rollback_failed_map(iova, size as u64); } + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. drop(mutation); - drop(vm_exec); - if retain { - core::mem::forget(node); - return Err(ret.err().unwrap_or(EIO)); - } + let _ = inner; ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// Unmap everything in an address range. @@ -2833,9 +2322,7 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; - // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; - // every subsequent access to VM data is serialized by the exec lock. - let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); let _commit = if user { @@ -2866,7 +2353,7 @@ impl Vm { let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), result => result?, }; @@ -2905,8 +2392,7 @@ impl Vm { // The BO's newest sequence is checked under the same lock as binds. If // it is not newer than the close, every mapping belongs to that close // and the per-object list is safe and substantially cheaper to walk. - // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { Err(error) if error == ENOMEM => { return self.drop_mappings_filtered(gem, sequence); } @@ -2999,18 +2485,8 @@ impl Drop for VmInner { let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let mut transaction = match self.uat_inner.lock_handoff() { - Ok(guard) => guard, - Err(_) => { - self.page_table.retain(); - pr_err!( - "MMU: retaining M4 VM {} page tables after PPL failure\n", - self.id - ); - return; - } - }; - let uat_inner = transaction.shared_mut(); + let uat_inner = self.uat_inner.lock(); + uat_inner.handoff().lock(); let handoff_cur = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); let inval = ttb_cur == ttb; @@ -3023,10 +2499,9 @@ impl Drop for VmInner { } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); - uat_inner.owned_user_roots[idx] = 0; - uat_inner.publish_slot(idx); } - drop(transaction); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); // In principle we dropped all the KernelMappings already, but we might as // well play it safe and invalidate the whole ASID. @@ -3046,7 +2521,7 @@ impl Drop for VmInner { /// Physical regions of a UAT whose handoff the generation has already established itself /// ([`Uat::new_adopted`]). None of them needs to be in the kernel linear map. -#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) struct AdoptedRegions { /// The handoff region (PPL magic, lock and flush requests). pub(crate) handoff: Range, @@ -3057,56 +2532,34 @@ pub(crate) struct AdoptedRegions { } impl Uat { - /// Common AP and firmware exclusion for G16's reserved-table window. Acquire - /// immutable root addresses before this guard, preserving binding -> UAT lock order. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn lock_m4_handoff(&self) -> Result> { - if !self.inner.m4_adopted { - return Err(EINVAL); - } - Ok(M4HandoffGuard { - _transaction: self.inner.lock_handoff()?, - }) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Map a region given by its physical range, write-combined. - fn map_physical( - range: &Range, - min_size: usize, - reserved_ram: bool, - ) -> Result { + fn map_physical(range: &Range, min_size: usize) -> Result { let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; - if size < min_size || range.start & (UAT_PGMSK as PhysicalAddr) != 0 { + if size < min_size || (range.start | range.end) & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); } // SAFETY: All-zero is a valid `struct resource` (no name, parent or children). let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as usize; + raw.flags = kernel::bindings::IORESOURCE_MEM as _; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; - let flags = io::mem::MemFlag::WC; - #[cfg(CONFIG_DRM_ASAHI_G15)] - let flags = if reserved_ram { - // G15 admits only whole reserved RAM pages or wholly non-RAM carveouts. - if crate::g15::memory_mode(range.start, size as u64)? - == kernel::bindings::MEMREMAP_WB as u32 - { - io::mem::MemFlag::WB - } else { - flags - } + // A carveout outside System RAM is mapped write-combined; a reserved-memory region + // inside System RAM (mapped by the kernel) through the kernel's own mapping, as + // `map_region` does for the other generations. + let mode = crate::g15::memory_mode(range.start, size as u64)?; + let ram = mode == kernel::bindings::MEMREMAP_WB as u32; + let flags = if ram { + io::mem::MemFlag::WB } else { - flags + io::mem::MemFlag::WC }; - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - let _ = reserved_ram; - // SAFETY: The UAT regions are only accessed through this mapping by the CPU; - // the GPU reaches them through its own translation. + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU + // reaches them through its own translation. let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, @@ -3114,7 +2567,7 @@ impl Uat { }) } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Creates a `Uat` for a handoff that the caller establishes itself (G15). /// /// This only maps the handoff and the TTBAT (write-combined); neither is written here. The @@ -3128,52 +2581,23 @@ impl Uat { /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after /// mapping and before the firmware uses the new addresses. Page tables are never freed on /// unmap, so linked entries stay valid. - #[allow(dead_code)] pub(crate) fn new_adopted( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, ) -> Result { - Self::new_adopted_inner(dev, geometry, regions, true, true) - } - - /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also - /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; - /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn new_adopted_with_user_upper( - dev: &driver::AsahiDrmDevice, - geometry: UatGeometry, - regions: &AdoptedRegions, - map_kernel_to_user: bool, - ) -> Result { - Self::new_adopted_inner(dev, geometry, regions, map_kernel_to_user, false) - } - - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] - fn new_adopted_inner( - dev: &driver::AsahiDrmDevice, - geometry: UatGeometry, - regions: &AdoptedRegions, - map_kernel_to_user: bool, - reserved_ram: bool, - ) -> Result { - let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE, reserved_ram)?; - let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE, reserved_ram)?; + let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; + let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { firmware: UatFirmware::Handoff, - m4_adopted: !map_kernel_to_user, - failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, - reserved: None, - quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3182,14 +2606,10 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], - binding_generations: [0; UAT_NUM_CTX], - binding_published: [false; UAT_NUM_CTX], - owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, - map_kernel_to_user, + map_kernel_to_user: true, handoff_rgn, ttbs_rgn, - quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -3201,14 +2621,8 @@ impl Uat { let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let shadow_root = kernel::page::Page::into_phys(shadow); - let kernel_lower_vm = Vm::new( - dev.into(), - inner.clone(), - geometry.user_range(), - geometry, - None, - 1, - )?; + let kernel_lower_vm = + Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; let kernel_vm = Vm::new( dev.into(), inner.clone(), @@ -3235,13 +2649,13 @@ impl Uat { }) } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Root of the kernel lower `Vm`, for TTBAT context 0 of an adopted handoff. pub(crate) fn kernel_lower_root(&self) -> PhysicalAddr { self.kernel_lower_vm.ttb() } - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Root whose driver-window entry names the second-level table of [`Self::kernel_vm`]'s /// driver window. With [`Self::new_adopted`] this is the shadow root that no context names. pub(crate) fn kernel_window_root(&self) -> PhysicalAddr { @@ -3305,16 +2719,13 @@ impl Uat { return Err(EINVAL); } let root = { - let owner = vm.gpuvm.lock_inner(); + let owner = lock_vm(&vm.gpuvm, None, false)?; if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { return Err(EINVAL); } owner.ttb() }; - Ok(self - .inner - .lock() - .contexts_naming_root(root, self.inner.m4_adopted)) + Ok(self.inner.lock().contexts_naming_root(root)) } /// Publish a completed alias set before its descriptors become visible. @@ -3343,24 +2754,12 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { - self.bind_inner(vm, true) - } - - /// Binds a VM without waiting for another slot owner to retire. - #[cfg(CONFIG_DRM_ASAHI_G15)] - pub(crate) fn try_bind(&self, vm: &Vm) -> Result { - self.bind_inner(vm, false) - } - - fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { - self.inner.check_m4()?; let mut binding = vm.binding.lock(); - self.inner.check_m4()?; if binding.binding.is_none() { assert_eq!(binding.active_users, 0); - let isolation = module_parameters::robust_isolation.value() != 0; + let isolation = *module_parameters::robust_isolation.value() != 0; self.slots.set_limit(if isolation { NonZeroUsize::new(1) @@ -3368,34 +2767,22 @@ impl Uat { None }); - let slot = if wait { - self.slots.get(binding.bind_token)? - } else { - self.slots.try_get(binding.bind_token)? - }; + let slot = self.slots.get(binding.bind_token)?; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let mut transaction = self.inner.lock_handoff()?; - let uat_inner = transaction.shared_mut(); - // On M4, empty bind/close churn must not wrap the wire generation - // back to firmware's last observed identity for a different root. - let previous = uat_inner.binding_generations[idx]; - let incarnation = if !self.inner.m4_adopted - || previous == 0 - || uat_inner.binding_published[idx] - { - previous.checked_add(1).ok_or(EOVERFLOW)? + let uat_inner = self.inner.lock(); + + let ttb1 = if uat_inner.map_kernel_to_user { + uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT } else { - previous + 0 }; - let ttb1 = - application_ttb1(uat_inner.kernel_ttb1, idx, uat_inner.map_kernel_to_user); - let ttbs = uat_inner.ttbs(); + uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { pr_err!( "Vm::bind to slot {}, but it is currently in use by the ASC?\n", @@ -3404,12 +2791,8 @@ impl Uat { } ttbs[idx].ttb0.store(ttb, Ordering::Release); ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.binding_generations[idx] = incarnation; - uat_inner.binding_published[idx] = false; - uat_inner.owned_user_roots[idx] = ttb; - binding.generation = binding_wire_generation(incarnation); - uat_inner.publish_slot(idx); - drop(transaction); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); // Make sure all TLB entries from the previous owner of this ASID are gone mem::sync(); @@ -3428,35 +2811,6 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } - /// Confirms the exact stopped-adopted M4 TTBAT contract. The same mutex serializes every - /// host root installation/removal; firmware does not write these words. An unexpected - /// observed root is an error, never a new ownership grant. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn check_adopted_bindings(&self) -> Result { - self.inner.check_m4()?; - // Binding locks precede the global UAT lock, as in bind(). - let lower = self.kernel_lower_vm.ttb() | TTBR_VALID; - let inner = self.inner.lock(); - self.inner.check_m4()?; - if self.inner.firmware != UatFirmware::Handoff || inner.map_kernel_to_user { - return Err(EINVAL); - } - let ttbs = inner.ttbs(); - if ttbs[0].ttb0.load(Ordering::Acquire) != lower - || ttbs[0].ttb1.load(Ordering::Acquire) != inner.kernel_ttb1 | TTBR_VALID - { - return Err(EIO); - } - for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { - if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] - || table.ttb1.load(Ordering::Acquire) != 0 - { - return Err(EIO); - } - } - Ok(()) - } - /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { @@ -3464,7 +2818,7 @@ impl Uat { return Err(EINVAL); } { - let owner = vm.gpuvm.lock_inner(); + let owner = lock_vm(&vm.gpuvm, None, false)?; if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) { @@ -3508,7 +2862,6 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); - inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -3518,17 +2871,14 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - self.inner.check_m4()?; - let vm = Vm::new( + Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, self.geometry, None, id, - )?; - self.inner.check_m4()?; - Ok(vm) + ) } /// Returns the address space geometry of this UAT. @@ -3550,19 +2900,14 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries( - self.inner.reserved.as_deref(), - self.kernel_vm.ttb(), - UAT_PGSZ / size_of::(), - |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }, - ) + pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }) }?; mem::sync(); @@ -3575,16 +2920,11 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries( - self.inner.reserved.as_deref(), - self.kernel_vm.ttb(), - snapshot.len(), - |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }, - ) + pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }) }?; if !matches { return Err(EIO); @@ -3596,19 +2936,15 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let Some(secondary) = self.inner.lock().secondary_ttb1 else { - // A single-instance firmware has no secondary root to mirror. - return Ok(()); - }; + let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; - let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(reserved, secondary, count, |secondary| { + pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -3625,33 +2961,24 @@ impl Uat { Ok(()) } - /// Discards the CPU's cached lines of the no-map table regions after the firmware built its - /// root there during boot (`UatQuirks::reserved_tables`; see `ReservedTables::invalidate`). - pub(crate) fn invalidate_reserved_tables(&self) { - if let Some(reserved) = self.inner.reserved.as_ref() { - reserved.invalidate(); - } - } - /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff_lock(); + inner.handoff().lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff_unlock(); + inner.handoff().unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.publish_slot(KERNEL_ALIAS_CTX); - inner.handoff_unlock(); + inner.handoff().unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -3661,30 +2988,21 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner( - dev: driver::AsahiDevice, - firmware: UatFirmware, - quirks: UatQuirks, - reserved: Option>, - ) -> Result> { + fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); - let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, - m4_adopted: false, - failed: AtomicBool::new(false), - clean_poc: quirks.clean_tables, - quirks, - reserved, + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + clean_poc: false, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3693,14 +3011,10 @@ impl Uat { kernel_ttb1: 0, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], - binding_generations: [0; UAT_NUM_CTX], - binding_published: [false; UAT_NUM_CTX], - owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, - quirks: shared_quirks, }, "uat_shared" ), @@ -3716,27 +3030,12 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, - quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - mem::set_page_tlbi(quirks.page_tlbi); - let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; - let reserved = if quirks.reserved_tables { - let mut tables = pgtable::ReservedTables::new(); - for name in [c"pagetables", c"l2-pagetables"] { - let res = of_node.reserved_mem_region_to_resource_byname(name)?; - // SAFETY: The boot loader reserved these regions (no-map) for the firmware's - // page tables; they stay reserved until reboot. - unsafe { tables.add(res)? }; - } - Some(Arc::new(tables, GFP_KERNEL)?) - } else { - None - }; - - let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let inner = Self::make_inner(dev.clone(), firmware)?; + let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -3744,11 +3043,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset: Some(offset), - } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), - UatFirmware::PublishedRoots { - secondary_root_offset: None, - } => (UAT_PGSZ, None), + secondary_root_offset, + } => ( + secondary_root_offset as usize + UAT_PGSZ, + Some(ttb1 + secondary_root_offset), + ), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -3799,18 +3098,11 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware, quirks.handoff_slot)?; - if quirks.clean_tables { - pgtable::clean_poc( - core::ptr::from_ref(inner.handoff()).cast::(), - HANDOFF_SIZE, - ); - pgtable::poc_barrier(); - } + inner.handoff().init(firmware)?; dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff_lock(); + inner.handoff().lock(); let ttbs = inner.ttbs(); @@ -3822,7 +3114,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if matches!(firmware, UatFirmware::PublishedRoots { .. }) { + if secondary_ttb1.is_some() { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3831,12 +3123,8 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } - if quirks.clean_tables { - pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); - pgtable::poc_barrier(); - } - inner.handoff_unlock(); + inner.handoff().unlock(); core::mem::drop(inner); @@ -3854,30 +3142,3 @@ impl Drop for Uat { mem::sync(); } } - -impl Vm { - /// Diagnostic reservation occupancy (including guards), not a placement promise. - /// Removal can progress between queries; allocations are excluded by the VM lock. - pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { - let inner = self.gpuvm.lock_inner(); - let mut next = range.start; - let mut used = 0; - let mut largest = 0; - while next < range.end { - let Some(node) = inner.mm.first_overlap(next..range.end)? else { - largest = largest.max(range.end - next); - break; - }; - largest = largest.max(node.start.saturating_sub(next)); - let end = node.end.min(range.end); - used += end - node.start.max(next); - next = end; - } - Ok((used, largest)) - } -} -impl Uat { - pub(crate) fn execution_contexts_used(&self) -> u32 { - (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() - } -} diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index acce5677b9f92..44a1575ecfd2b 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -17,7 +17,6 @@ pub(crate) struct PreparedUserMap { single_page: bool, context: context::PreparedBinding, _residency: Option, - _sg_lease: kernel::drm::gem::shmem::SGLease, } pub(crate) struct PreparedUserUnmap { @@ -58,7 +57,7 @@ impl PreparedUserBindBatch { } impl Vm { - /// Prepares host state before taking the GPUVM exec lock or changing any leaf. + /// Prepares host state without taking DMA reservations or changing any leaf. pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, @@ -81,20 +80,24 @@ impl Vm { // SAFETY: As in `Vm::bind_object()`, the device is bound while its file can issue binds. let dev = unsafe { self.dev.as_bound() }; // Populate the scatter-gather table before any GPUVM lock is taken. - let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + crate::gem::held_sg_table(gem, dev)?; let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &sg_lease)?; + { + let mut segments = vm_bo.data().segments.lock(); + if segments.is_none() { + segments.replace(prefetch_sgt_segments(dev, gem)?); + } + } if (addr | size | offset) & UAT_PGMSK as u64 != 0 { return Err(EINVAL); } Ok(PreparedUserMap { _residency: residency, - _sg_lease: sg_lease, gem: gem.into(), vm_bo, resources, @@ -135,7 +138,7 @@ impl Vm { PreparedUserBindOp::Map(map) => { // SAFETY: See `lock_vm()`. As in `Vm::bind_object()`, the object itself is // not locked: the map step may still populate its pages. - let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; + let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; match self.mapping_commit() { Ok(commit) => { let (flags, repeat) = if map.single_page { @@ -185,7 +188,7 @@ impl Vm { } PreparedUserBindOp::Unmap(unmap) => { // SAFETY: See `lock_vm()`. - let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; + let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; match self.mapping_commit() { Ok(commit) => { let mut ctx = diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 66d507c2d7694..30010be497973 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -79,13 +79,9 @@ pub(super) struct VmLifetime { impl VmLifetime { pub(super) fn can_reclaim(&self) -> bool { self.state.try_lock().is_some_and(|state| { - state.active == 0 - && state.commits == 0 - && !state.blocked - && !state.quarantined - && !state.draining - && !state.closed - && !state.cleanup_failed + state.active == 0 && state.commits == 0 + && !state.blocked && !state.quarantined && !state.draining + && !state.closed && !state.cleanup_failed }) } @@ -343,12 +339,10 @@ impl Vm { drop(previous); } - /// Whether the client's handle of this VM is gone ([`Self::unmap_user_ranges`] closed - /// admission for good). G15 releases its per-VM state after that. + /// Whether userspace has permanently closed this VM's map admission. + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn is_closed(&self) -> bool { - self.lifetime - .as_ref() - .is_some_and(|lifetime| lifetime.state.lock().closed) + self.lifetime.as_ref().is_some_and(|lifetime| lifetime.state.lock().closed) } /// Tests mappings and reservations which userspace must never replace. @@ -417,10 +411,7 @@ impl Vm { } if !state.draining { state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(VmJobGuard { - vm: self.clone(), - _residency: residency, - }); + return Ok(VmJobGuard { vm: self.clone(), _residency: residency }); } lifetime.changed.wait(&mut state); } @@ -480,11 +471,7 @@ impl Vm { Ok(()) } - pub(super) fn defer_bind_batch( - &self, - batch: &mut PreparedUserBindBatch, - start: usize, - ) -> Result { + pub(super) fn defer_bind_batch(&self, batch: &mut PreparedUserBindBatch, start: usize) -> Result { use super::bind::PreparedUserBindOp; let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); @@ -498,8 +485,7 @@ impl Vm { } if batch .ops - .iter() - .skip(start) + .iter().skip(start) .any(|op| matches!(op, PreparedUserBindOp::Map(_))) { return Err(EIO); @@ -510,21 +496,16 @@ impl Vm { // Reserve before accepting any range; a partial deferred batch must // not leave canonical metadata ahead of the accepted cleanup. state.unmaps.reserve(batch.ops.len() - start, GFP_KERNEL)?; - let retired = - self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { - PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), - PreparedUserBindOp::Map(_) => None, - })); + let retired = self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); // Transfer the preallocated split nodes along with each accepted range. // Already-committed prefix operations must not be replayed: their nodes // may have been consumed and another binding may now occupy the range. for op in batch.ops.drain(start..) { if let PreparedUserBindOp::Unmap(unmap) = op { - if !state - .unmaps - .iter() - .any(|old| old.iova == unmap.iova && old.size == unmap.size) - { + if !state.unmaps.iter().any(|old| old.iova == unmap.iova && old.size == unmap.size) { state.unmaps.push(unmap, GFP_KERNEL)?; } } @@ -549,30 +530,16 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } - if state - .unmaps - .iter() - .any(|old| old.iova == iova && old.size == size) - { + if state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { return Ok(true); } drop(state); let unmap = self.prepare_user_unmap(iova, size)?; let mut state = lifetime.state.lock(); - if state.closed { - return Err(ENOENT); - } - if !state.blocked { - return Ok(false); - } - if !state.quarantined { - return Err(EBUSY); - } - if !state - .unmaps - .iter() - .any(|old| old.iova == iova && old.size == size) - { + if state.closed { return Err(ENOENT); } + if !state.blocked { return Ok(false); } + if !state.quarantined { return Err(EBUSY); } + if !state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { state.unmaps.push(unmap, GFP_KERNEL)?; } Ok(true) @@ -667,8 +634,8 @@ impl Vm { let mut failure = None; loop { let close = state.close_ranges.take(); - let unmaps = core::mem::take(&mut state.unmaps); - let objects = core::mem::take(&mut state.objects); + let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); + let objects = core::mem::replace(&mut state.objects, KVec::new()); drop(state); if let Some((user, kernel)) = close { if let Err(error) = self.unmap_user_ranges_now(user, kernel) { @@ -704,9 +671,7 @@ impl Vm { // terminal failure instead of silently discarding its outcome. state.blocked = true; state.cleanup_failed = true; - if let Some(status) = &self.status { - status.record(error); - } + if let Some(status) = &self.status { status.record(error); } } else if !state.closed && !state.cleanup_failed { state.blocked = false; } diff --git a/drivers/gpu/drm/asahi/mmu/ppl.rs b/drivers/gpu/drm/asahi/mmu/ppl.rs deleted file mode 100644 index 7cdad5e1f3831..0000000000000 --- a/drivers/gpu/drm/asahi/mmu/ppl.rs +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! The M4 two-party PPL protocol. The transport owns the host mutex for the entire -//! guard lifetime: firmware exclusion alone does not serialize two AP threads. -//! This core has no kernel dependencies so the production guard can be exercised -//! against deterministic firmware and concurrent host transports. - -pub(super) trait Memory { - fn firmware_interested(&self) -> bool; - fn firmware_turn(&self) -> bool; - fn ap_interest(&self, interested: bool); - fn give_firmware_turn(&self); - fn barrier(&self); - fn expired(&self) -> bool; - fn relax(&self); - /// Latch failure before the transport releases host exclusion on timeout. - fn fail(&self); -} - -/// Owns both the host-side lock (in `memory`) and successful firmware exclusion. -#[must_use = "retain the guard until every protected table mutation has completed"] -pub(super) struct Guard { - memory: T, -} - -impl Guard { - pub(super) fn acquire(memory: T) -> Result { - let timeout = || { - memory.ap_interest(false); - memory.barrier(); - memory.fail(); - Err(()) - }; - memory.ap_interest(true); - memory.barrier(); - while memory.firmware_interested() { - if memory.expired() { - return timeout(); - } - if memory.firmware_turn() { - memory.ap_interest(false); - memory.barrier(); - while memory.firmware_turn() { - if memory.expired() { - return timeout(); - } - memory.relax(); - } - memory.ap_interest(true); - memory.barrier(); - } - memory.relax(); - } - memory.barrier(); - Ok(Self { memory }) - } - - pub(super) fn memory_mut(&mut self) -> &mut T { - &mut self.memory - } -} - -impl Drop for Guard { - fn drop(&mut self) { - self.memory.give_firmware_turn(); - self.memory.barrier(); - self.memory.ap_interest(false); - self.memory.barrier(); - // Only after these stores complete may `memory` release the host mutex. - } -} diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 1df852184b31d..5370b87ccf1b5 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -251,8 +251,6 @@ pub(crate) struct VmShrinker { // SAFETY: shrinker core serializes registration lifetime; callback VM operations // use its residency/reservation locks. raw is never dereferenced after free. unsafe impl Send for VmShrinker {} -// SAFETY: Shared callbacks use the VM's residency/reservation locks. The -// registration remains live until shrinker_free has drained all callbacks. unsafe impl Sync for VmShrinker {} impl VmShrinker { @@ -266,7 +264,7 @@ impl VmShrinker { )?; // SAFETY: The format is static and has no format arguments. let raw = - unsafe { kernel::bindings::shrinker_alloc(0, c_str!("aurora-idle-vm").as_char_ptr()) }; + unsafe { kernel::bindings::shrinker_alloc(0, c_str!("asahi-idle-vm").as_char_ptr()) }; if raw.is_null() { return Err(ENOMEM); } @@ -275,7 +273,7 @@ impl VmShrinker { unsafe { (*raw).count_objects = Some(Self::count); (*raw).scan_objects = Some(Self::scan); - (*raw).private_data = core::ptr::from_ref(&*owner).cast_mut().cast(); + (*raw).private_data = (&*owner as *const Self).cast_mut().cast(); (*raw).seeks = 2; (*raw).batch = 256; kernel::bindings::shrinker_register(raw); @@ -283,36 +281,23 @@ impl VmShrinker { Ok(owner) } - /// Count reclaimable pages. - /// - /// # Safety - /// - /// `raw` must be this owner's live registered shrinker. The core must - /// retain its private data until the callback returns. unsafe extern "C" fn count( raw: *mut kernel::bindings::shrinker, _sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: The registration owns this stable allocation until callbacks drain. let owner = unsafe { &*((*raw).private_data as *const Self) }; - owner.vm.idle_reclaim_count() + owner.vm.idle_reclaim_count() as _ } - /// Reclaim idle pages. - /// - /// # Safety - /// - /// `raw` must be this owner's live registered shrinker, and `sc` must - /// point to the initialized control supplied by the shrinker core. unsafe extern "C" fn scan( raw: *mut kernel::bindings::shrinker, sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: Both pointers are supplied by the registered shrinker core. let owner = unsafe { &*((*raw).private_data as *const Self) }; - // SAFETY: The shrinker core supplies a live initialized control. - let target = unsafe { (*sc).nr_to_scan }; - owner.vm.reclaim_idle_pages(target) + let target = unsafe { (*sc).nr_to_scan } as usize; + owner.vm.reclaim_idle_pages(target) as _ } } @@ -363,7 +348,7 @@ impl Vm { if self .status .as_ref() - .is_none_or(|status| status.get() != 0) + .map_or(true, |status| status.get() != 0) { return 0; } @@ -376,17 +361,11 @@ impl Vm { // this reservation; linked aliases retain their borrowed BO and GEM. let _ = unsafe { inner.for_each_private_bo(|_, bo| { let object = bo.obj(); - // Count retained backing on idle VMs, and purgeable contents even if idle - // reclaim already released the table. Never wait on the BO mutex under resv. - let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - let advised = object.madv_locked() > 0; - let releasable = if segments.is_some() { - object.sole_sg_lease_releasable_locked() - } else { - (advised || object.has_sg_table_locked()) - && object.unleased_pages_releasable_locked() - }; - if object.idle_reclaim_candidate() && (advised || idle) && releasable { + // Purgeable objects count on busy VMs too; idle backing only on idle ones. + if object.idle_reclaim_candidate() + && (object.madv_locked() > 0 || idle) + && object.sole_sg_lease_releasable_locked() + { pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(true) @@ -406,7 +385,7 @@ impl Vm { if self .status .as_ref() - .is_none_or(|status| status.get() != 0) + .map_or(true, |status| status.get() != 0) { return 0; } @@ -417,7 +396,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return 0; }; - let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); + let contexts = shared.contexts_naming_root(inner.ttb()); let mut purged = 0usize; // SAFETY: The private reservation excludes binding edits and restoration. // Callbacks change PTEs/flags only; all aliases stay linked. @@ -429,12 +408,9 @@ impl Vm { } { let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - let releasable = if segments.is_some() { - object.sole_sg_lease_releasable_locked() - } else { - object.unleased_pages_releasable_locked() - }; - if !releasable { return Ok(true); } + if segments.is_some() && !object.sole_sg_lease_releasable_locked() { + return Ok(true); + } } // Any mutation that does not end in a completed purge leaves // INVALIDATED aliases with cleared PTEs while madv stays >= 0, so @@ -491,7 +467,7 @@ impl Vm { if self .status .as_ref() - .is_none_or(|status| status.get() != 0) + .map_or(true, |status| status.get() != 0) || !self .lifetime .as_ref() @@ -506,7 +482,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return purged; }; - let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); + let contexts = shared.contexts_naming_root(inner.ttb()); let mut unpinned = 0usize; // SAFETY: Exclusive residency and the private reservation exclude jobs, // binding edits and retirement. Callbacks change PTEs/flags only; all @@ -518,14 +494,9 @@ impl Vm { { // Avoid inversion against preparation (VmBoInner -> resv). let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - let releasable = if segments.is_some() { - object.sole_sg_lease_releasable_locked() - } else { - // A competing lease may have made the previous release fail after we - // dropped our segments. Retry that table, but do not count empty backing. - object.has_sg_table_locked() && object.unleased_pages_releasable_locked() - }; - if !releasable { return Ok(true); } + if segments.is_none() || !object.sole_sg_lease_releasable_locked() { + return Ok(true); + } } // The association retains SG backing until *all* aliases have // been invalidated and their device translations flushed. A diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index fec7a6b61ad8f..b9e2975c898ff 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -63,7 +63,7 @@ impl Vm { Binding { start: address, end, - object: core::ptr::from_ref(gem) as usize, + object: gem as *const gem::Object as usize, offset, bind_sequence, }, @@ -99,7 +99,7 @@ impl Vm { if shared.count.load(Ordering::Relaxed) == 0 { return; } - let object = core::ptr::from_ref(gem) as usize; + let object = gem as *const gem::Object as usize; let mut bindings = shared.bindings.lock(); bindings.retain(|entry| entry.object != object || entry.bind_sequence > sequence); shared.count.store(bindings.len() as u32, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index 0c9e6f62b5bbe..38a2268137eff 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -152,9 +152,9 @@ macro_rules! inner_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p.cast::(); + let inner = p as *const u8; // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer.cast::()) }; + let off = unsafe { inner.offset_from(outer as *const u8) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -234,9 +234,9 @@ macro_rules! inner_weak_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p.cast::(); + let inner = p as *const u8; // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer.cast::()) }; + let off = unsafe { inner.offset_from(outer as *const u8) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -298,15 +298,11 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc - .ptr() - .ok_or(EINVAL)? - .as_ptr() - .cast::>(); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; let mut raw = callback(&inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is // identical to the type of `raw` other than the lifetime. - unsafe { p.copy_from(core::ptr::from_mut(&mut raw).cast::().cast(), 1) }; + unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; mem::forget(raw); Ok(Self { raw: p, @@ -341,14 +337,10 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc - .ptr() - .ok_or(EINVAL)? - .as_ptr() - .cast::>>(); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. let raw = callback(&inner, unsafe { &mut *p })?; - if !core::ptr::eq(p.cast::>(), raw) { + if p as *mut T::Raw<'_> != raw as *mut _ { dev_err!( alloc.device().as_ref(), "Allocation callback returned a mismatched reference ({})\n", @@ -357,7 +349,7 @@ impl> GpuObject { return Err(EINVAL); } Ok(Self { - raw: p.cast::().cast::>(), + raw: p as *mut u8 as *mut T::Raw<'static>, gpu_ptr, alloc, inner, @@ -408,16 +400,16 @@ impl> GpuObject { alloc.gpu_ptr() ); let inner = inner_init(gpu_ptr); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr().cast::>(); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; let ret = Self { - raw: p.cast::().cast::>(), + raw: p as *mut u8 as *mut T::Raw<'static>, gpu_ptr, alloc, inner: KBox::init(inner, GFP_KERNEL)?, }; - let q = core::ptr::from_ref::(&*ret.inner); + let q = &*ret.inner as *const T; // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - unsafe { pin_init::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; + unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; Ok(ret) } @@ -445,7 +437,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&mut *self.raw, &mut *core::ptr::from_mut(&mut *self.inner)) } + unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } } /// Access the inner `Raw` data given a user-supplied callback. @@ -457,7 +449,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&*self.raw, &*core::ptr::from_ref(&*self.inner)) } + unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } } } @@ -483,8 +475,8 @@ where f.debug_struct(core::any::type_name::()) // SAFETY: `self.raw` is valid per the type invariant. .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) - .field("inner", &format_args!("{:#X?}", self.inner)) - .field("alloc", &format_args!("{:?}", self.alloc)) + .field("inner", &format_args!("{:#X?}", &self.inner)) + .field("alloc", &format_args!("{:?}", &self.alloc)) .finish() } } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 0f915607cb1f4..b88353a2f7344 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -18,8 +18,7 @@ use kernel::{ addr::PhysicalAddr, error::Result, page::Page, - prelude::*, - sync::Arc, // + prelude::*, // }; use crate::debug::*; @@ -54,20 +53,19 @@ const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; -/// The system-wide minimum CPU line size, including heterogeneous CPUs. A local -/// CTR_EL0 read can select too large a stride if the task migrates during the loop. +/// Size of the smallest CPU data cache line (`CTR_EL0.DminLine`). #[inline(always)] fn dcache_line_size() -> usize { - use kernel::bindings; - let ctr = (bindings::SYS_CTR_EL0_Op0 << 19) - | (bindings::SYS_CTR_EL0_Op1 << 16) - | (bindings::SYS_CTR_EL0_CRn << 12) - | (bindings::SYS_CTR_EL0_CRm << 8) - | (bindings::SYS_CTR_EL0_Op2 << 5); - // SAFETY: Read-only lookup of the arm64 sanitized feature register. DminLine - // uses the architecture's LOWER_SAFE merge policy across all admitted CPUs. - let value = unsafe { bindings::read_sanitised_ftr_reg(ctr) }; - 4 << ((value >> bindings::CTR_EL0_DminLine_SHIFT) & 0xf) + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1 and reading it has no side effects. + unsafe { + core::arch::asm!( + "mrs {ctr}, ctr_el0", + ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags) + ) + }; + 4 << ((ctr >> 16) & 0xf) } /// Cleans the CPU data cache lines covering `len` bytes at `ptr` to the point of coherency, for an @@ -97,6 +95,7 @@ pub(crate) fn clean_poc(ptr: *const u8, len: usize) { /// Cleans and invalidates the CPU data cache lines covering `len` bytes at `ptr` to the point of /// coherency, so that later CPU reads observe what a non-snooping agent wrote. Lines the CPU wrote /// are written back first. [`poc_barrier`] completes the maintenance. +#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { if len == 0 { return; @@ -117,130 +116,20 @@ pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { } } -/// Completes earlier point-of-coherency maintenance for the whole system (`dsb sy`). +/// Completes earlier point-of-coherency maintenance for the outer shareable domain (`dsb osh`). pub(crate) fn poc_barrier() { // SAFETY: Barriers are always safe. - unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; -} - -/// Page tables in bootloader-reserved memory that the kernel does not map ("no-map"), reached -/// through a write-back mapping instead of `struct page` (the `pagetables` and `l2-pagetables` -/// regions, where the firmware builds its root and kernel-window tables). -pub(crate) struct ReservedTables { - regions: KVec, - starts: KVec, -} - -// SAFETY: The mappings are plain write-back memory owned for the lifetime of this object; table -// entries are only accessed as atomics. -unsafe impl Send for ReservedTables {} -// SAFETY: As above. -unsafe impl Sync for ReservedTables {} - -impl ReservedTables { - pub(crate) fn new() -> Self { - Self { - regions: KVec::new(), - starts: KVec::new(), - } - } - - /// Adds a reserved region, mapped write-back. - /// - /// # Safety - /// - /// `res` must be a bootloader-reserved region outside System RAM that stays reserved. - pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { - let start = res.start(); - // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. - let map = - unsafe { kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? }; - self.starts.push(start, GFP_KERNEL)?; - self.regions.push(map, GFP_KERNEL)?; - Ok(()) - } - - /// The CPU address of a page-table page in one of the regions. - fn lookup(&self, phys: PhysicalAddr) -> Option<*mut u8> { - for (start, map) in self.starts.iter().zip(self.regions.iter()) { - let Some(offset) = phys.checked_sub(*start) else { - continue; - }; - let offset = offset as usize; - if offset + UAT_PGSZ <= map.size() { - // SAFETY: The offset and one page lie inside the mapping. - return Some(unsafe { map.ptr().add(offset) }); - } - } - None - } - - /// Whether `phys` is a page of one of the regions. - pub(crate) fn contains(&self, phys: PhysicalAddr) -> bool { - self.lookup(phys).is_some() - } - - /// Drops the CPU's cached lines of every region, so that later reads see what the firmware - /// wrote while booting. Lines the host dirtied are written back first, never discarded. - /// - /// This runs once, after the firmware booted. The host has written table entries by then - /// (fixed kernel objects are mapped before the boot), but the T6040, the only SoC with - /// reserved tables, also sets `UatQuirks::clean_tables`: every host table store is cleaned - /// to the PoC as it is made, so this drops only clean lines and loses no host write. - /// - /// The regions are not invalidated again, which assumes the firmware no longer writes the - /// tables the host walks once it has booted. That is not proven by a trace; the J616s has - /// run this way since its GPU came up without a fault traced to a stale table. - pub(crate) fn invalidate(&self) { - for map in self.regions.iter() { - clean_invalidate_poc(map.ptr(), map.size()); - } - poc_barrier(); - } -} - -/// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the -/// reserved mapping when the page lies in a reserved region, else through `struct page`. With -/// reserved regions, a page outside them that is not System RAM with a `struct page` (other -/// no-map memory) is refused with `EIO`. -/// -/// # Safety -/// -/// `phys` must be a page-table page that stays allocated while `f` runs. -unsafe fn with_table( - reserved: Option<&ReservedTables>, - phys: PhysicalAddr, - offset: usize, - len: usize, - f: impl FnOnce(*mut u8) -> Result, -) -> Result { - if let Some(base) = reserved.and_then(|r| r.lookup(phys)) { - if offset.checked_add(len).ok_or(EINVAL)? > UAT_PGSZ { - return Err(EINVAL); - } - // SAFETY: lookup() checked that the whole page is mapped. - return f(unsafe { base.add(offset) }); - } - let page = if reserved.is_some() { - // SAFETY: The caller guarantees the page stays allocated; borrow_phys() checks that it - // is System RAM with a struct page. - unsafe { Page::borrow_phys(&phys) }.ok_or(EIO)? - } else { - // SAFETY: The caller guarantees an allocated page-table page in RAM. - unsafe { Page::borrow_phys_unchecked(&phys) } - }; - page.with_pointer_into_page(offset, len, f) + unsafe { core::arch::asm!("dsb osh", options(nostack, preserves_flags)) }; } /// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. /// /// # Safety /// -/// `ttb` must be the physical address of a page table root, in RAM or in one of the `reserved` -/// regions, that stays allocated while `f` runs. Other agents (the GPU firmware) may access the -/// entries concurrently, which is why they are only exposed as atomics. +/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` +/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they +/// are only exposed as atomics. pub(crate) unsafe fn with_root_entries( - reserved: Option<&ReservedTables>, ttb: PhysicalAddr, count: usize, f: impl FnOnce(&[AtomicU64]) -> R, @@ -248,18 +137,15 @@ pub(crate) unsafe fn with_root_entries( if count > UAT_NPTE { return Err(EINVAL); } - // SAFETY: A root outside the reserved regions must have a struct page. - if !reserved.is_some_and(|r| r.contains(ttb)) && unsafe { Page::borrow_phys(&ttb) }.is_none() { - return Err(EIO); - } - // SAFETY: The caller guarantees an allocated root page, in RAM or in a reserved region. - unsafe { - with_table(reserved, ttb, 0, count * PTE_SIZE, |p| { - // SAFETY: with_table() checked that `count` entries are mapped, and page table entries - // are naturally aligned. - Ok(f(core::slice::from_raw_parts(p as *const AtomicU64, count))) - }) - } + // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. + let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; + page.with_pointer_into_page(0, count * PTE_SIZE, |p| { + // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page + // table entries are naturally aligned. + Ok(f(unsafe { + core::slice::from_raw_parts(p as *const AtomicU64, count) + })) + }) } const PTE_TYPE_BITS: u64 = 3; @@ -367,9 +253,7 @@ impl Prot { Self { memattr, ..*self } } - /// The protection bits of a leaf with this protection (no address, table type or non-global - /// bits), for checking that a firmware leaf word round-trips through [`Self::from_pte`]. - #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) const fn leaf_bits(&self) -> u64 { self.as_pte() } @@ -394,22 +278,6 @@ impl Prot { (!need_read || readable) && (!need_write || writable) } - /// The native form of a GPU-only permission: AP 0 with the access in PXN (read) and UXN - /// (write), as [`Self::allows_gpu`] reads AP 0. The logical selectors RO, WO, RW, NA map to - /// the native RO, WO, RW, NA. Firmware permissions are returned unchanged. - const fn native_gpu(self) -> Self { - if self.ap != AP_GPU { - return self; - } - let selector = (self.high_bits >> 1) & 3; - Self { - ap: AP_FW_GPU, - high_bits: (self.high_bits & !(HIGH_BITS_PXN | HIGH_BITS_UXN)) - | (((selector + 1) & 3) << 1), - ..self - } - } - pub(crate) const fn is_cached_noncoherent(&self) -> bool { self.ap != AP_GPU && self.memattr == MEMATTR_CACHED } @@ -428,12 +296,6 @@ impl Default for Prot { pub(crate) struct UatPageTable { ttb: PhysicalAddr, ttb_owned: bool, - /// Ownership could not be relinquished to firmware; physical tables remain until reboot. - retained: bool, - /// Reserved regions holding tables this page table may reach. - reserved: Option>, - /// Store GPU-only permissions in the native AP 0 form ([`Prot::native_gpu`]). - native_gpu: bool, /// Tables below a root the driver did not allocate may belong to the bootloader or the /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the /// tables below the root, as with the handoff firmware interface. @@ -443,6 +305,7 @@ pub(crate) struct UatPageTable { oas_mask: u64, /// Clean new tables and changed entries to the point of coherency, for a UAT walker that does /// not snoop the CPU caches ([`Self::enable_clean_poc`]). + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: bool, } @@ -454,49 +317,45 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: true, - retained: false, - reserved: None, - native_gpu: false, driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } /// Makes every later change of this page table visible to a UAT walker that does not snoop the - /// CPU caches (G16, and `UatQuirks::clean_tables`), and cleans the root, which may have been - /// zeroed through the cacheable linear map. + /// CPU caches (G15), and cleans the root, which was zeroed through the cacheable linear map. /// /// From now on, a new table page is cleaned before its parent descriptor names it, a changed /// table or leaf descriptor is cleaned after the store, and every mutating walk ends with a - /// `dsb sy`. Call it before anything is mapped and before any context names the root. + /// `dsb osh`. Call it before anything is mapped and before any context names the root. + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn enable_clean_poc(&mut self) -> Result { self.clean_poc = true; - // SAFETY: The root is owned or borrowed by this page table for its lifetime, in RAM or in - // a reserved region. - unsafe { - with_table(self.reserved.as_deref(), self.ttb, 0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - }) - }?; + // SAFETY: The root is a RAM page owned or borrowed by this page table for its lifetime. + let root = unsafe { Page::borrow_phys(&self.ttb) }.ok_or(EIO)?; + root.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; poc_barrier(); Ok(()) } - /// Whether changes are cleaned to the point of coherency. + /// Whether changes are cleaned to the point of coherency (always false without UAT adoption support). #[inline(always)] fn cleans_poc(&self) -> bool { - self.clean_poc - } - - /// Stores GPU-only permissions of this (client or kernel-lower) table in the native AP 0 form - /// (`UatQuirks::native_gpu_permissions`). Firmware roots never hold GPU-only mappings. - pub(crate) fn with_native_gpu(mut self, native_gpu: bool) -> Self { - self.native_gpu = native_gpu; - self + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + { + self.clean_poc + } + #[cfg(not(CONFIG_DRM_ASAHI_UAT_ADOPTION))] + { + false + } } /// Borrow a root; track individual child ownership only when other agents @@ -507,7 +366,6 @@ impl UatPageTable { ias: u32, oas: u32, shared_tables: bool, - reserved: Option>, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -523,9 +381,7 @@ impl UatPageTable { return Err(EINVAL); } // SAFETY: The bootloader-managed root stays valid while this page table exists. - let reachable = reserved.as_ref().is_some_and(|r| r.contains(ttb)) - || unsafe { Page::borrow_phys(&ttb) }.is_some(); - if !reachable { + if unsafe { Page::borrow_phys(&ttb) }.is_none() { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", ttb @@ -536,23 +392,15 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: false, - retained: false, - reserved, - native_gpu: false, driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } - /// Retains every physical table after an uncertain M4 handoff. No other agent may reclaim - /// this tree merely because a userspace handle closed or a fence reported an error. - pub(crate) fn retain(&mut self) { - self.retained = true; - } - pub(crate) fn ttb(&self) -> PhysicalAddr { self.ttb } @@ -622,7 +470,6 @@ impl UatPageTable { // Handle the case where iova_range.end is just at the top boundary of the IAS let end = ((iova_range.end - 1) & self.ias_mask) + 1; - let reserved = self.reserved.clone(); let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); // For every child table, retain the parent page and entry that names it. // Children are detached before their parents are freed, so these borrowed @@ -666,61 +513,55 @@ impl UatPageTable { ); let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) as usize; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); - let driver_tables = &mut self.driver_tables; - let oas_mask = self.oas_mask; - let entry = |p: *mut u8| -> Result> { - let uptep = p.cast_const().cast::(); - // SAFETY: with_table() ensures the pointer is valid, and our index is - // aligned so it is safe to deref as an AtomicU64. - let upte = unsafe { &*uptep }; - let mut upte_val = upte.load(Ordering::Relaxed); - // Allocate if requested - if upte_val == 0 && alloc { - let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!( - "UATPageTable::with_pages: alloc PT at {:#x}\n", - pt_page.phys() - ); - if clean { - // A non-snooping walker must never follow a parent to a - // table whose zeroes are still in the CPU caches. - pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; + pt_addr[level] = + upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_pointer_into_page() ensures the pointer is valid, + // and our index is aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if clean { + // A non-snooping walker must never follow a parent to a + // table whose zeroes are still in the CPU caches. + pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; + } + if let Some(tables) = self.driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; + } + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); + upte.store(upte_val, Ordering::Relaxed); + if clean { + clean_poc(uptep.cast(), PTE_SIZE); + } } - if let Some(tables) = driver_tables.as_mut() { - tables.push(pt_page.phys(), GFP_KERNEL)?; + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) } - let pt_paddr = Page::into_phys(pt_page); - upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; - // Complete child-table initialization before a - // concurrent device walk can follow its parent. - // A later mapping/TLBI barrier cannot order an - // already-visible parent against earlier clears. - crate::mem::sync(); - upte.store(upte_val, Ordering::Relaxed); - if clean { - clean_poc(uptep.cast(), PTE_SIZE); - } - } - if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) - } else if upte_val == 0 || (!alloc && !free) { - mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); - Ok(None) - } else { - pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, - iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); - Ok(None) - } - }; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed tables in RAM or in a reserved region. - pt_addr[level] = unsafe { - with_table(reserved.as_deref(), phys, upidx * PTE_SIZE, PTE_SIZE, entry) - }?; + })?; if pt_addr[level].is_some() { pt_parent[level] = Some((phys, upidx)); } @@ -756,10 +597,13 @@ impl UatPageTable { count, iova ); - let leaves = |p: *mut u8| -> Result { - let ptep = p.cast_const().cast::(); + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; + pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let ptep = p as *const _ as *const Pte; // SAFETY: We know this is a valid pointer to PTEs and the range is valid and - // checked by with_table(). + // checked by with_pointer_into_page(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; let result = cb(iova, ptes); if clean && mutate { @@ -767,18 +611,7 @@ impl UatPageTable { clean_poc(p, count * PTE_SIZE); } result - }; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed tables in RAM or in a reserved region. - unsafe { - with_table( - reserved.as_deref(), - phys, - idx * PTE_SIZE, - count * PTE_SIZE, - leaves, - ) - }?; + })?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); iova = align(iova + 1, block); @@ -803,9 +636,16 @@ impl UatPageTable { } /// Free a page table, unless it was not allocated by the driver. - fn free_table(&mut self, phys: PhysicalAddr, parent: Option<(PhysicalAddr, usize)>) -> Result { + fn free_table( + &mut self, + phys: PhysicalAddr, + parent: Option<(PhysicalAddr, usize)>, + ) -> Result { let owned = if let Some(tables) = self.driver_tables.as_ref() { - tables.contains(&phys) + match tables.iter().position(|&table| table == phys) { + Some(_) => true, + None => false, + } } else { true }; @@ -813,55 +653,32 @@ impl UatPageTable { return Ok(()); } if let Some((parent_phys, parent_index)) = parent { - let oas_mask = self.oas_mask; - let in_reserved = self - .reserved - .as_ref() - .is_some_and(|r| r.contains(parent_phys)); - // SAFETY: Only looks the page up; nothing is accessed through it. - if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { - return Err(EIO); - } - let reserved = self.reserved.as_deref(); - let cleans = self.cleans_poc(); - let detach = |p: *mut u8| -> Result { - // SAFETY: The live page-table page contains aligned u64 PTEs, and with_table() - // bounds this access to one PTE. + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; + page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { let pte = unsafe { &*(p as *const AtomicU64) }; let current = pte.load(Ordering::Acquire); - let expected = phys & oas_mask & (!UAT_PGMSK as u64); - if current & oas_mask & (!UAT_PGMSK as u64) != expected { + let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); + if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { return Err(EIO); } pte.store(0, Ordering::Release); - if cleans { + if self.cleans_poc() { clean_poc(p, PTE_SIZE); } Ok(()) - }; - // SAFETY: The bottom-up traversal keeps the parent allocated while - // its child descriptor is detached. A changed descriptor indicates - // an ownership violation; fail closed instead of freeing a table - // that may still be reachable. - unsafe { - with_table( - reserved, - parent_phys, - parent_index * PTE_SIZE, - PTE_SIZE, - detach, - ) - }?; + })?; } if let Some(tables) = self.driver_tables.as_mut() { - let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; + let index = tables + .iter() + .position(|&table| table == phys) + .ok_or(EIO)?; tables.swap_remove(index); } - if self.cleans_poc() { - // Complete the parent descriptor clear before its detached child can be reused. - // Callers have already removed this tree from live contexts before freeing tables. - poc_barrier(); - } // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without // `driver_tables`, every child table belongs to the driver. unsafe { Page::from_phys(phys) }; @@ -952,15 +769,6 @@ impl UatPageTable { } } - /// The permission bits as this table stores them. - fn stored(&self, prot: Prot) -> Prot { - if self.native_gpu { - prot.native_gpu() - } else { - prot - } - } - pub(crate) fn map_pages( &mut self, iova_range: Range, @@ -980,7 +788,6 @@ impl UatPageTable { } let pte_bits = self.pte_bits(); - let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { @@ -1007,7 +814,6 @@ impl UatPageTable { iova_range, prot ); - let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); @@ -1055,9 +861,6 @@ impl UatPageTable { impl Drop for UatPageTable { fn drop(&mut self) { - if self.retained { - return; - } mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); if self .with_pages(self.va_range.clone(), false, true, |iova, ptes| { diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs index 20396f2e77225..a68352828cfbc 100644 --- a/drivers/gpu/drm/asahi/queue/common.rs +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -33,7 +33,7 @@ pub(super) fn get_timestamp_object( return Err(ERANGE); } Ok(Some(UserTimestamp { - mapping: mapping.firmware_mapping()?, + mapping: mapping.clone(), offset: offset as usize, })) } else { diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index aa5dc1fba4723..0872a50ce5d46 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -67,9 +67,7 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - if layouts != [0; 2] { - return Err(EINVAL); - } + if layouts != [0; 2] { return Err(EINVAL); } Ok(()) } @@ -153,7 +151,6 @@ pub(crate) struct QueueInner { notifier_list: Arc>, notifier: Arc>, usc_exec_base: u64, - usage: Arc, id: u64, #[ver(V >= V13_0B4)] counter: AtomicU64, @@ -194,7 +191,7 @@ impl JobFence::ver { #[vtable] impl dma_fence::FenceOps for JobFence::ver { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("aurora") + c_str!("asahi") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("queue") @@ -265,14 +262,10 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { type TimeoutData = QueueTimeout; - const MODULE: Option<&'static kernel::ThisModule> = - Some(kernel::module::this_module::()); + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); fn timeout_data(&self) -> QueueTimeout { - QueueTimeout { - dev: self.dev.clone(), - id: self.id, - } + QueueTimeout { dev: self.dev.clone(), id: self.id } } fn prepare(job: &mut Self) -> Option { @@ -364,7 +357,7 @@ impl sched::JobImpl for QueueJob::ver { return Err(ENODEV); }; - guard.registration_data_with(|reg_data| { + return guard.registration_data_with(|reg_data| { let gpu = match reg_data .gpu .clone() @@ -439,7 +432,7 @@ impl sched::JobImpl for QueueJob::ver { core::mem::drop(frag_job); Ok(Some(Fence::from_fence(&job.fence))) - }) + }); } fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { @@ -468,7 +461,7 @@ impl Drop for QueueJob::ver { } } -static QUEUE_NAME: &CStr = c_str!("aurora_fence"); +static QUEUE_NAME: &CStr = c_str!("asahi_fence"); static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); #[versions(AGX)] @@ -488,7 +481,6 @@ impl Queue::ver { id: u64, priority: u32, usc_exec_base: u64, - usage: Arc, ) -> Result { mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); @@ -526,21 +518,12 @@ impl Queue::ver { WQ_SIZE, 0, 100000, - c_str!("aurora_sched"), + c_str!("asahi_sched"), )?; let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; let buffer = buffer::Buffer::ver::new(gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; - let gpu_context = Arc::new( - workqueue::GpuContext::new(dev.clone(), alloc, buffer.any_ref())?, - GFP_KERNEL, - )?; - // Every command of this queue points the firmware at the notifier, and the notifier links - // into the notifier list of the work queues. Keep it as long as they are kept, until the - // context is invalidated, instead of freeing it with the last command. - gpu_context.retain(KBox::new(notifier.clone(), GFP_KERNEL)?)?; - let mut ret = Queue::ver { dev: dev.clone(), _sched: sched, @@ -554,12 +537,15 @@ impl Queue::ver { inner: QueueInner::ver { dev: dev.clone(), ualloc, - gpu_context, + gpu_context: Arc::new( + workqueue::GpuContext::new(dev.clone(), alloc, buffer.any_ref())?, + GFP_KERNEL, + )?, + buffer, notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, notifier, usc_exec_base, - usage, id, #[ver(V >= V13_0B4)] counter: AtomicU64::new(0), @@ -567,7 +553,7 @@ impl Queue::ver { }; // Rendering structures - let tvb_blocks = module_parameters::initial_tvb_size.value(); + let tvb_blocks = *module_parameters::initial_tvb_size.value(); ret.inner.buffer.ensure_blocks(tvb_blocks)?; @@ -637,11 +623,8 @@ const COMPUTE_SUPPORTED_SIZE: usize = // SAFETY: All bit patterns are valid by construction. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} -// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} -// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_compute {} -// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_attachment {} fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { @@ -652,10 +635,8 @@ fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result, size: usize) -> Result> 7; - attachments.list[i] = microseq::Attachment { + attachments.list[i as usize] = microseq::Attachment { address: U64(att.pointer), size: cache_lines.try_into()?, unk_c: 0x17, @@ -773,33 +754,10 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; - let payload_size = header.size as usize; - match u32::from(header.cmd_type) { - uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { - let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; - if render - .ts_frag_end_copies - .iter() - .any(|ts| ts.handle != 0 || ts.offset != 0) - { - return Err(EINVAL); - } - } - uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { - let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; - if compute - .ts_end_copies - .iter() - .any(|ts| ts.handle != 0 || ts.offset != 0) - { - return Err(EINVAL); - } - } - _ => cmdbuf.skip(payload_size), - } + cmdbuf.skip(header.size as usize); nr_commands += 1; - match u32::from(header.cmd_type) { + match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { last_render = nr_commands; nr_render += 1; @@ -891,7 +849,7 @@ impl Queue for Queue::ver { EINVAL })? { let mut alloc = gpu.alloc(); - let queue_job = match u32::from(header.cmd_type) { + let queue_job = match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, _ => return Err(EINVAL), @@ -921,7 +879,7 @@ impl Queue for Queue::ver { } } - match u32::from(header.cmd_type) { + match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf .read_supported(header_size, RENDER_SUPPORTED_SIZE) diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index e8b845cba989f..fb22569ea5fdf 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -13,15 +13,12 @@ use crate::hw; use core::mem::MaybeUninit; use kernel::{ - bindings, - c_str, + bindings, c_str, device::Core, + devres::Devres, error::{from_err_ptr, to_result}, io::{ - mem::{ - DevresIoMem, - IoMem, // - }, + mem::IoMem, Io, IoBase, // }, @@ -225,10 +222,7 @@ impl CpuControl { }; // SAFETY: pdev is live; the provider establishes the consumer device link. let mbox = unsafe { - from_err_ptr(bindings::apple_mbox_get( - pdev.as_ref().as_raw(), - index.try_into()?, - ))? + from_err_ptr(bindings::apple_mbox_get(pdev.as_ref().as_raw(), index.try_into()?))? }; let mut lifecycle = MaybeUninit::uninit(); // SAFETY: mbox is live, and lifecycle is writable storage for the C result. @@ -241,10 +235,7 @@ impl CpuControl { } // SAFETY: This only checks the provider's matching start/stop capability. to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(mbox) })?; - Ok(Self { - dev: pdev.into(), - mbox, - }) + Ok(Self { dev: pdev.into(), mbox }) } /// Start through the owner of the CPU registers and runtime-PM reference. @@ -260,11 +251,7 @@ impl CpuControl { let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }); if let Err(error) = result { if error != ENODEV { - dev_err!( - self.dev.as_ref(), - "Could not stop GPU coprocessor: {:?}\n", - error - ); + dev_err!(self.dev.as_ref(), "Could not stop GPU coprocessor: {:?}\n", error); } } } @@ -566,17 +553,14 @@ impl Resources<'_> { #[derive(Clone)] pub(crate) struct G17Resources { dev: ARef, - sgx: Arc>, + sgx: Arc>>, } impl G17Resources { /// Map the required resources given our platform device. pub(crate) fn new(pdev: &platform::Device>) -> Result { let sgx_req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; - let sgx = Arc::new( - sgx_req.iomap_sized::()?.into_devres()?, - GFP_KERNEL, - )?; + let sgx = Arc::new(sgx_req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?; Ok(G17Resources { dev: pdev.into(), @@ -709,11 +693,6 @@ impl G17Resources { self.sgx_read32::() & 0xf != 0 } - /// Raw core power state, for tracing only. - pub(crate) fn g17_core_power(&self) -> u32 { - self.sgx_read32::() - } - /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { @@ -722,6 +701,7 @@ impl G17Resources { let val = sgx.try_read32(G17_GATING_CONTROL)?; sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) } + } /// Split the packed per-cluster core masks into one mask per cluster. @@ -752,7 +732,11 @@ fn split_core_masks( } if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!(dev.as_ref(), "Leftover core mask: {:#x?}\n", core_mask_regs); + dev_err!( + dev.as_ref(), + "Leftover core mask: {:#x?}\n", + core_mask_regs + ); return Err(EIO); } @@ -760,10 +744,7 @@ fn split_core_masks( } /// Decode the silicon revision from the ID version register. -fn gpu_revision( - dev: &platform::Device, - id_version: u32, -) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { +fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { Ok(match (id_version >> 8) & 0xff { 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), @@ -777,29 +758,3 @@ fn gpu_revision( } }) } - -/// Device-managed SGX mapping retained by the G16 firmware session. -#[cfg(CONFIG_DRM_ASAHI_G16)] -#[derive(Clone)] -pub(crate) struct G16Resources { - sgx: Arc>, -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -impl G16Resources { - pub(crate) fn new(pdev: &platform::Device>) -> Result { - let req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; - Ok(Self { - sgx: Arc::new(req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?, - }) - } - pub(crate) fn sgx_read32_at(&self, offset: usize) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - (&*sgx).as_view().try_read32(offset) - } - #[allow(dead_code)] - pub(crate) fn sgx_write32_at(&self, offset: usize, value: u32) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - (&*sgx).as_view().try_write32(value, offset) - } -} diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 7132e6de41c09..9f0baa4350d93 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -8,8 +8,6 @@ #include #include #include -#include -#include #include #include #include @@ -18,7 +16,7 @@ #include "sysfs.h" -/* Legacy prefix of repr(C) Rust atomics; READ_ONCE accesses current values. */ +/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ struct asahi_stats_snapshot { u32 util1; u32 util2; @@ -34,85 +32,12 @@ struct asahi_stats_snapshot { u64 jobs; }; -/* Mirror of the repr(C) Rust `memacct::Ledger`, in bytes. */ -struct asahi_gpu_memory { - u64 user_committed; - u64 user_resident; - u64 kernel_resident; - u64 refused; - u64 limit; - u64 client_limit; -}; - struct asahi_stats_attribute { struct device_attribute attr; - struct device_attribute memory_attr; const struct asahi_stats_snapshot *snapshot; - const struct asahi_gpu_memory *memory; bool export_enabled; - asahi_stats_read_fn read; - asahi_stats_suspend_fn suspend; - struct notifier_block pm_notifier; }; -/* 0: unavailable, 1: valid, 2: stale; matches g17::telemetry. */ -static const char *asahi_stats_quality(u32 status) -{ - return status == 1 ? "valid" : status == 2 ? "stale" : "unavailable"; -} - -static ssize_t agx_qualified_stats_show(struct asahi_stats_attribute *stats, char *buf) -{ - struct asahi_qualified_stats snap; - ssize_t n = 0; - - stats->read(stats->snapshot, &snap); - n += sysfs_emit_at(buf, n, "utilisation_status %s\n", - asahi_stats_quality(snap.util_status)); - if (snap.util_status == 1) { - n += sysfs_emit_at(buf, n, "util_tiler_percent %u\n", snap.tiler); - n += sysfs_emit_at(buf, n, "util_renderer_percent %u\n", snap.renderer); - n += sysfs_emit_at(buf, n, "util_device_percent %u\n", snap.device); - } else { - n += sysfs_emit_at(buf, n, "util_tiler_percent unsupported\n" - "util_renderer_percent unsupported\n" - "util_device_percent unsupported\n"); - } - n += sysfs_emit_at(buf, n, "pstate_status %s\n", - asahi_stats_quality(snap.pstate_status)); - if (snap.pstate_status == 1) - n += sysfs_emit_at(buf, n, "pstate %u\n", snap.pstate); - else - n += sysfs_emit_at(buf, n, "pstate unsupported\n"); - /* No A/B selector, packet duration, measured power or temperature. */ - n += sysfs_emit_at(buf, n, "requested_frequency_hz unsupported\n" - "achieved_frequency_hz unsupported\n" - "busy_ns unsupported\n" - "power_mw unsupported\n" - "temperature_raw unsupported\n"); - return n; -} - -static int asahi_stats_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) -{ - struct asahi_stats_attribute *stats = - container_of(nb, struct asahi_stats_attribute, pm_notifier); - - switch (action) { - case PM_SUSPEND_PREPARE: - case PM_HIBERNATION_PREPARE: - case PM_RESTORE_PREPARE: - stats->suspend(stats->snapshot, true); - break; - case PM_POST_SUSPEND: - case PM_POST_HIBERNATION: - case PM_POST_RESTORE: - stats->suspend(stats->snapshot, false); - break; - } - return NOTIFY_OK; -} - static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -123,8 +48,6 @@ static ssize_t agx_stats_show(struct device *dev, if (!stats->export_enabled) return sysfs_emit(buf, "unsupported\n"); - if (stats->read) - return agx_qualified_stats_show(stats, buf); n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", (unsigned long long)READ_ONCE(snap->busy_ns)); n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", @@ -149,56 +72,20 @@ static ssize_t agx_stats_show(struct device *dev, return n; } -/* - * GPU memory held by the driver, in bytes: user objects created (committed) - * and held by the GPU (resident), held kernel objects, the limits that admit - * user objects (0: none) and the number of refused creations. - */ -static ssize_t gpu_memory_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct asahi_stats_attribute *stats = - container_of(attr, struct asahi_stats_attribute, memory_attr); - const struct asahi_gpu_memory *mem = stats->memory; - ssize_t n = 0; - - n += scnprintf(buf + n, PAGE_SIZE - n, "user_committed %llu\n", - (unsigned long long)READ_ONCE(mem->user_committed)); - n += scnprintf(buf + n, PAGE_SIZE - n, "user_resident %llu\n", - (unsigned long long)READ_ONCE(mem->user_resident)); - n += scnprintf(buf + n, PAGE_SIZE - n, "kernel_resident %llu\n", - (unsigned long long)READ_ONCE(mem->kernel_resident)); - n += scnprintf(buf + n, PAGE_SIZE - n, "limit %llu\n", - (unsigned long long)READ_ONCE(mem->limit)); - n += scnprintf(buf + n, PAGE_SIZE - n, "client_limit %llu\n", - (unsigned long long)READ_ONCE(mem->client_limit)); - n += scnprintf(buf + n, PAGE_SIZE - n, "refused %llu\n", - (unsigned long long)READ_ONCE(mem->refused)); - - return n; -} /* * Called from Rust's `AsahiDriver::probe` after the DRM device is * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle, - asahi_stats_read_fn read, asahi_stats_suspend_fn suspend) + int export_enabled, void **handle) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || !memory || ((export_enabled || read) && !snapshot) || - (!!read != !!suspend)) + if (!dev || !handle || (export_enabled && !snapshot)) return -ENODEV; - BUILD_BUG_ON(offsetof(struct asahi_qualified_stats, util_host_ns) != 24); - BUILD_BUG_ON(sizeof(struct asahi_qualified_stats) != 56); - BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); - BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); - BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); - BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 40); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); @@ -214,36 +101,9 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->attr.show = agx_stats_show; stats->snapshot = snapshot; stats->export_enabled = export_enabled; - stats->read = read; - stats->suspend = suspend; - if (suspend) { - stats->pm_notifier.notifier_call = asahi_stats_pm_notify; - ret = register_pm_notifier(&stats->pm_notifier); - if (ret) { - kfree(stats); - return ret; - } - } - - sysfs_attr_init(&stats->memory_attr.attr); - stats->memory_attr.attr.name = "gpu_memory"; - stats->memory_attr.attr.mode = 0444; - stats->memory_attr.show = gpu_memory_show; - stats->memory = memory; ret = device_create_file(dev, &stats->attr); if (ret) { - if (stats->suspend) - unregister_pm_notifier(&stats->pm_notifier); - kfree(stats); - return ret; - } - - ret = device_create_file(dev, &stats->memory_attr); - if (ret) { - device_remove_file(dev, &stats->attr); - if (stats->suspend) - unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -253,46 +113,6 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, } EXPORT_SYMBOL_GPL(asahi_sysfs_register); -/* - * G17: GPU idle power-off delay (ms, 2-1000). On T6040/25G83 the firmware powers the GPU cores - * off after 2 ms idle and reads its own delay only at boot; above 2 the driver holds the cores - * powered until the GPU has been idle this long (power::IDLE_HOLD_MS). Each wake-up there costs - * about 1 ms, so userspace may lengthen it on AC power. - */ -static u32 *asahi_idle_word; - -static ssize_t idle_off_delay_ms_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - return sysfs_emit(buf, "%u\n", READ_ONCE(*asahi_idle_word)); -} - -static ssize_t idle_off_delay_ms_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - u32 ms; - - if (kstrtou32(buf, 0, &ms) || ms < 2 || ms > 1000) - return -EINVAL; - WRITE_ONCE(*asahi_idle_word, ms); - return count; -} - -static DEVICE_ATTR_RW(idle_off_delay_ms); - -int asahi_sysfs_register_idle(struct device *dev, u32 *word) -{ - int ret; - - if (!dev || !word) - return -ENODEV; - asahi_idle_word = word; - ret = device_create_file(dev, &dev_attr_idle_off_delay_ms); - if (ret) - asahi_idle_word = NULL; - return ret; -} -EXPORT_SYMBOL_GPL(asahi_sysfs_register_idle); - void asahi_sysfs_unregister(struct device *dev, void *handle) { struct asahi_stats_attribute *stats = handle; @@ -300,20 +120,12 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) if (!dev || !stats) return; - /* Both notifier removal and file removal join readers of the Rust Arc. */ - if (stats->suspend) - unregister_pm_notifier(&stats->pm_notifier); /* Removal drains active show callbacks before either allocation dies. */ - device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); - if (asahi_idle_word) { - device_remove_file(dev, &dev_attr_idle_off_delay_ms); - asahi_idle_word = NULL; - } kfree(stats); } EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); MODULE_AUTHOR("AGX driver maintainer"); MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("AGX firmware stats sysfs shim"); +MODULE_DESCRIPTION("Asahi AGX firmware stats sysfs shim"); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index d43f482903e55..5969a1718925f 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -3,31 +3,10 @@ #ifndef _ASAHI_SYSFS_H #define _ASAHI_SYSFS_H -#include - struct device; -/* Mirrors g17::telemetry::Readout, filled under the Rust device mutex. */ -struct asahi_qualified_stats { - u32 util_status; - u32 pstate_status; - u32 tiler; - u32 renderer; - u32 device; - u32 pstate; - u64 util_host_ns; - u64 pstate_host_ns; - u64 util_samples; - u64 pstate_timestamp; -}; - -typedef void (*asahi_stats_read_fn)(const void *snapshot, struct asahi_qualified_stats *out); -typedef void (*asahi_stats_suspend_fn)(const void *snapshot, bool suspended); - int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle, - asahi_stats_read_fn read, asahi_stats_suspend_fn suspend); -int asahi_sysfs_register_idle(struct device *dev, u32 *word); + int export_enabled, void **handle); void asahi_sysfs_unregister(struct device *dev, void *handle); #endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index 7c34f0627b020..079623d1ef210 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,111 +3,24 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::{g17::telemetry::Readout, memacct::Ledger, stats::StatsSnapshot}; +use crate::stats::StatsSnapshot; use core::ffi::{c_int, c_void}; -type ReadStats = unsafe extern "C" fn(*const c_void, *mut Readout); -type SuspendStats = unsafe extern "C" fn(*const c_void, bool); - extern "C" { fn asahi_sysfs_register( dev: *mut c_void, - snapshot: *const c_void, + snapshot: *const StatsSnapshot, export_enabled: c_int, - memory: *const Ledger, handle: *mut *mut c_void, - read: Option, - suspend: Option, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } -extern "C" { - fn asahi_sysfs_register_idle(dev: *mut c_void, word: *mut u32) -> c_int; -} - -/// Export the idle power-off delay as the read-write `idle_off_delay_ms` (G17). -pub(crate) fn register_idle( - dev: *mut kernel::bindings::device, - word: *mut u32, -) -> kernel::error::Result { - // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes - // it with READ_ONCE/WRITE_ONCE. - let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; - if ret < 0 { - Err(kernel::error::Error::from_errno(ret)) - } else { - Ok(()) - } -} - /// Register only after initialization succeeds, keeping the snapshot Arc alive. -/// -/// Also registers `gpu_memory`, which shows the static GPU memory ledger. pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, export_enabled: bool, -) -> kernel::error::Result<*mut c_void> { - register_with_callbacks(dev, snapshot, export_enabled, None, None) -} - -/// The caller retains the G17 snapshot Arc until unregister joins both sysfs -/// readers and the PM notifier. The callbacks never power the GPU up. -pub(crate) fn register_g17( - dev: *mut kernel::bindings::device, - snapshot: *const StatsSnapshot, - export_enabled: bool, -) -> kernel::error::Result<*mut c_void> { - register_with_callbacks( - dev, - snapshot, - export_enabled, - Some(read_g17), - Some(suspend_g17), - ) -} - -/// Copy the locked qualified snapshot to the C formatter's local output. -/// -/// # Safety -/// -/// `snapshot` must name a live StatsSnapshot retained by the registration. -/// `out` must be aligned and writable for one Readout for this call. -unsafe extern "C" fn read_g17(snapshot: *const c_void, out: *mut Readout) { - // SAFETY: C calls only while registration retains the snapshot Arc, and - // supplies a writable, aligned output for this call. - let snapshot = unsafe { &*snapshot.cast::() }; - let view = snapshot - .g17_reader - .as_ref() - .map_or_else(Readout::default, |reader| reader.read()); - // SAFETY: As above; the C output is initialized in one locked read. - unsafe { out.write(view) }; -} - -/// End the telemetry epoch at either side of system sleep. -/// -/// # Safety -/// -/// `snapshot` must name a live StatsSnapshot. The caller must join this -/// callback before releasing the registration's owning Arc. -unsafe extern "C" fn suspend_g17(snapshot: *const c_void, suspended: bool) { - // SAFETY: The notifier is unregistered and joined before the Arc is dropped. - if let Some(reader) = unsafe { &*snapshot.cast::() } - .g17_reader - .as_ref() - { - reader.suspend(suspended); - } -} - -fn register_with_callbacks( - dev: *mut kernel::bindings::device, - snapshot: *const StatsSnapshot, - export_enabled: bool, - read: Option, - suspend: Option, ) -> kernel::error::Result<*mut c_void> { let mut handle = core::ptr::null_mut(); // SAFETY: The bound driver keeps the device and optional snapshot alive until @@ -115,12 +28,9 @@ fn register_with_callbacks( let ret = unsafe { asahi_sysfs_register( dev.cast(), - snapshot.cast(), + snapshot, c_int::from(export_enabled), - &crate::memacct::LEDGER, &mut handle, - read, - suspend, ) }; if ret < 0 { diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs deleted file mode 100644 index 9f13436b4caf5..0000000000000 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ /dev/null @@ -1,220 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Prepared timestamp destinations shared by native command completion paths. - -use crate::mmu; -#[cfg(CONFIG_DRM_ASAHI_G16)] -use crate::{file, gem}; -#[cfg(CONFIG_DRM_ASAHI_G16)] -use core::ops::Range; -#[cfg(CONFIG_DRM_ASAHI_G16)] -use core::sync::atomic::{fence as barrier, Ordering}; -use kernel::{prelude::*, sync::Arc}; -#[cfg(CONFIG_DRM_ASAHI_G16)] -use kernel::{uapi, xarray}; - -/// A firmware-mapped timestamp buffer with a kernel CPU mapping of its backing, kept for the -/// buffer's lifetime so that commands need no mapping of their own. -pub(crate) struct FirmwareBuffer { - mapping: Arc, - cpu: mmu::TimestampVmap, -} - -/// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. -/// CPU-retired timestamps retain their backing without consuming global firmware address space. -pub(crate) enum Buffer { - /// The last reference may be a command's, dropped where fence producers run, while - /// releasing the CPU mapping takes the reservation lock: the deferral runs the release - /// on the cleanup queue. - Firmware(crate::cleanup::Deferred), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Cpu { - // The GEM pin keeps the object's held SG table alive through CPU publication. - gem: gem::KernelMappingPin, - range: Range, - }, -} - -impl Buffer { - pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { - let cpu = mapping.timestamp_vmap()?; - Ok(Self::Firmware(crate::cleanup::Deferred::new(FirmwareBuffer { - mapping: Arc::new(mapping, GFP_KERNEL)?, - cpu, - })?)) - } - - /// Retain a CPU-written byte range without publishing any firmware address. - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn cpu(bo: gem::ObjectRef, range: Range) -> Result { - let size = range.end.checked_sub(range.start).ok_or(EINVAL)?; - if size == 0 || (range.start | size) & mmu::UAT_PGMSK != 0 || range.end > bo.size() { - return Err(EINVAL); - } - let gem = gem::KernelMappingPin::new(&bo.gem); - let registered = bo.gem.dev().registration_guard().ok_or(ENODEV)?; - let parent: &kernel::platform::Device = registered.as_ref(); - let _ = gem::held_sg_table(&gem, parent.as_ref())?; - Ok(Self::Cpu { gem, range }) - } - - pub(crate) fn size(&self) -> usize { - match self { - Self::Firmware(buffer) => buffer.mapping.size(), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Self::Cpu { range, .. } => range.end - range.start, - } - } - - #[cfg(CONFIG_DRM_ASAHI_G16)] - pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { - match self { - Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Self::Cpu { gem, range, .. } => { - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { - return Err(ERANGE); - } - mmu::TimestampMapping::new(gem, range.start.checked_add(offset).ok_or(EOVERFLOW)?) - } - } - } - - /// The object offset of the timestamp at `offset` of a firmware buffer, for - /// [`Self::write_timestamp`]. - pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { - match self { - Self::Firmware(buffer) => buffer.mapping.timestamp_offset(offset), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Self::Cpu { .. } => Err(EINVAL), - } - } - - /// Writes a timestamp at an object offset returned by [`Self::timestamp_offset`], - /// through the buffer's own CPU mapping. - pub(crate) fn write_timestamp(&self, offset: usize, value: u64) { - match self { - Self::Firmware(buffer) => buffer.cpu.write(offset, value), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Self::Cpu { .. } => {} - } - } - - /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. - pub(crate) fn firmware_mapping(&self) -> Result> { - match self { - Self::Firmware(buffer) => Ok(buffer.mapping.clone()), - #[cfg(CONFIG_DRM_ASAHI_G16)] - Self::Cpu { .. } => Err(EINVAL), - } - } -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -fn checked_offset(offset: u64, size: usize, aligned: bool) -> Result { - let offset = usize::try_from(offset).map_err(|_| EINVAL)?; - let end = offset.checked_add(size_of::()).ok_or(EINVAL)?; - if aligned && offset % size_of::() != 0 { - return Err(EINVAL); - } - if end > size { - return Err(if aligned { EINVAL } else { ERANGE }); - } - Ok(offset) -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -struct Destination { - // Retain the bound range and backing along with its prepared CPU view. - _mapping: Arc, - cpu: mmu::TimestampMapping, -} - -#[cfg(CONFIG_DRM_ASAHI_G16)] -impl Destination { - fn resolve( - objects: Pin<&xarray::XArray>>, - timestamp: uapi::drm_asahi_timestamp, - aligned: bool, - ) -> Result> { - if timestamp.handle == 0 { - return Ok(None); - } - let object = objects - .lock() - .get(timestamp.handle.try_into()?) - .ok_or(ENOENT)? - .clone(); - let file::Object::TimestampBuffer(mapping) = object; - let offset = checked_offset(timestamp.offset.into(), mapping.size(), aligned)?; - let cpu = mapping.timestamp_mapping(offset)?; - Ok(Some(Self { - _mapping: mapping, - cpu, - })) - } - - fn write(&self, ticks: u64, frequency_hz: u64) { - const NSEC_PER_SEC: u64 = 1_000_000_000; - let ns = (ticks / frequency_hz) - .wrapping_mul(NSEC_PER_SEC) - .wrapping_add(ticks % frequency_hz * NSEC_PER_SEC / frequency_hz); - self.cpu.write(ns); - barrier(Ordering::Release); - } -} - -/// Timestamp mappings resolved and retained before any command is enqueued. -#[cfg(CONFIG_DRM_ASAHI_G16)] -pub(crate) struct Destinations(Option; 4]>>); - -#[cfg(CONFIG_DRM_ASAHI_G16)] -impl Destinations { - /// G16 retains its eight-byte destination alignment and EINVAL contract. - pub(crate) fn resolve_aligned( - objects: Pin<&xarray::XArray>>, - timestamps: &[uapi::drm_asahi_timestamps], - ) -> Result { - Self::resolve_inner(objects, timestamps, true) - } - - fn resolve_inner( - objects: Pin<&xarray::XArray>>, - timestamps: &[uapi::drm_asahi_timestamps], - aligned: bool, - ) -> Result { - if timestamps.is_empty() || timestamps.len() > 2 { - return Err(EINVAL); - } - let mut destinations = core::array::from_fn(|_| None); - for (index, timestamp) in timestamps.iter().enumerate() { - destinations[2 * index] = Destination::resolve(objects, timestamp.start, aligned)?; - destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end, aligned)?; - } - // Empty submissions need no cleanup allocation. Timestamped commands - // prepare the unique work item before any output fence is published. - let retained = if destinations.iter().any(Option::is_some) { - Some(crate::cleanup::Deferred::new(destinations)?) - } else { - None - }; - Ok(Self(retained)) - } - - /// Publish measured timestamps before signalling completion or releasing dependent work. - pub(crate) fn publish(&self, ticks: [u64; 4], frequency_hz: u64) { - for (destination, ticks) in self.iter().zip(ticks) { - if let Some(destination) = destination { - destination.write(ticks, frequency_hz); - } - } - } - - fn iter(&self) -> core::slice::Iter<'_, Option> { - let destinations: &[Option] = self - .0 - .as_deref() - .map_or(&[], |destinations| destinations.as_slice()); - destinations.iter() - } -} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index 6e5540caeac3f..b51b027eb573c 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -96,13 +96,6 @@ pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { m } -/// A value that can be populated from an untrusted byte record. -/// -/// # Safety -/// -/// Every bit pattern must be valid for Self. Its byte representation must -/// contain no uninitialized bytes (including padding) when produced by Default, -/// since Reader temporarily views that storage as a byte slice. pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} pub(crate) struct Reader<'a> { @@ -124,9 +117,7 @@ impl<'a> Reader<'a> { // SAFETY: The output pointer is valid, and the size does not exceed // the type size, and all bit patterns are valid. - let dst = unsafe { - core::slice::from_raw_parts_mut(core::ptr::from_mut(&mut obj).cast::(), size) - }; + let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; dst.copy_from_slice(&src[..size]); self.offset = end; diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 4b613b1cc7f7a..c00da634972e5 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -87,16 +87,8 @@ static int __drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_obj * why this is required _and_ expected if you're * going to pin these pages. */ - gfp_t gfp = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN; - - /* A bulk pin must not run direct reclaim once per folio when - * the driver can instead return an allocation failure. Clearing - * DIRECT_RECLAIM also makes memcg charges fail without reclaim - * or invoking the OOM killer. Preserve placement and kswapd. - */ - if (shmem->pages_no_reclaim) - gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_RETRY_MAYFAIL); - mapping_set_gfp_mask(obj->filp->f_mapping, gfp); + mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER | + __GFP_RETRY_MAYFAIL | __GFP_NOWARN); } return 0; @@ -491,23 +483,10 @@ void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, } EXPORT_SYMBOL_GPL(drm_gem_shmem_vunmap_locked); -/** - * drm_gem_shmem_create_with_handle - Allocate an object with the given size and - * returns a GEM handle - * @file_priv: DRM file structure to create the dumb buffer for - * @dev: DRM device - * @size: Size of the object to allocate - * @handle: Returns the GEM handle on success - * - * Allocates an shmem GEM buffer using drm_gem_shmem_create() and returns - * a GEM handle to it. - * - * Returns: - * Zero on success, or an error code otherwise. - */ -int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, - struct drm_device *dev, size_t size, - uint32_t *handle) +static int +drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + uint32_t *handle) { struct drm_gem_shmem_object *shmem; int ret; @@ -526,7 +505,6 @@ int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, return ret; } -EXPORT_SYMBOL_GPL(drm_gem_shmem_create_with_handle); /* Update madvise status, returns true if not purged, else * false or -errno. @@ -634,10 +612,10 @@ bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shm struct drm_gem_object *obj = &shmem->base; dma_resv_assert_held(obj->resv); - return !drm_gem_is_imported(obj) && !obj->dma_buf && + return shmem->sgt && !drm_gem_is_imported(obj) && !obj->dma_buf && shmem->madv >= 0 && !refcount_read(&shmem->pages_pin_count) && !refcount_read(&shmem->vmap_use_count) && - refcount_read(&shmem->pages_use_count) == !!shmem->sgt && + refcount_read(&shmem->pages_use_count) == 1 && dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP); } EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); @@ -1002,7 +980,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; } - if (vma_is_cow_mapping(vma)) + if (is_cow_mapping(vma->vm_flags)) return -EINVAL; #ifdef CONFIG_MM_SUBPAGE @@ -1037,7 +1015,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = vma_get_page_prot(vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); if (shmem->map_wc) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); @@ -1097,10 +1075,6 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ if (shmem->sgt) return shmem->sgt; - /* Purged contents cannot be restored by rebuilding an empty table. */ - if (shmem->madv < 0) - return ERR_PTR(-EINVAL); - drm_WARN_ON(obj->dev, drm_gem_is_imported(obj)); ret = drm_gem_shmem_get_pages_locked(shmem); diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index 8392f9e4a0109..d25bca5352aa6 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -37,13 +37,6 @@ struct drm_gem_shmem_object { /** @reclaimable_cpu_mappings: CPU VMAs may be revoked by the driver. */ bool reclaimable_cpu_mappings; - /** - * @pages_no_reclaim: Disable direct reclaim during backing population. - * Set before drm_gem_shmem_init(). Drivers that can fail a large pin - * request cheaply may opt in; background reclaim remains enabled. - */ - bool pages_no_reclaim; - /** * @cpu_pages: Pages populated one at a time by CPU faults on revocable * VMAs while no device mapping holds @pages. Sparse; under base.resv. @@ -152,9 +145,7 @@ int drm_gem_shmem_madvise_locked(struct drm_gem_shmem_object *shmem, int madv); static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem) { return (shmem->madv > 0) && - !refcount_read(&shmem->pages_pin_count) && - !refcount_read(&shmem->vmap_use_count) && - refcount_read(&shmem->pages_use_count) == !!shmem->sgt && + !refcount_read(&shmem->pages_pin_count) && shmem->sgt && !shmem->base.dma_buf && !drm_gem_is_imported(&shmem->base); } @@ -168,10 +159,6 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem) void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem, struct drm_printer *p, unsigned int indent); -int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, - struct drm_device *dev, size_t size, - uint32_t *handle); - extern const struct vm_operations_struct drm_gem_shmem_vm_ops; /* diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 418522205ee6f..d311b92620b86 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -212,20 +212,6 @@ struct drm_asahi_params_global { * DRM_ASAHI_GPU_HAL_200. */ __u32 gpu_hal_generation; - - /** - * @memory_limit: Device-wide limit on user committed plus kernel - * resident bytes. Zero means no driver limit. This is a capacity, - * not current free memory; other clients share it. - */ - __u64 memory_limit; - - /** - * @client_memory_limit: Per-DRM-file limit on user committed bytes. - * Zero means no driver limit. Userspace must zero-initialize the reply - * buffer: older kernels leave these optional tail fields untouched. - */ - __u64 client_memory_limit; }; /** @@ -362,66 +348,6 @@ enum drm_asahi_feature { * accept render_scs_layout. */ DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 15, - - /** - * @DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD: Compute commands accept - * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. - */ - DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, - - /* - * Bits 17 to 28 belong to the M5 (G17G/G17S) runtime, which reports - * them; this kernel never does. They are listed so that the numbers are - * not given to anything else, and userspace reads every kernel's - * features the same way. Bits 25 and 27 are also reserved for that - * runtime. - */ - - /** @DRM_ASAHI_FEATURE_RENDER_RESOURCES: M5: per-program render storage. */ - DRM_ASAHI_FEATURE_RENDER_RESOURCES = (1UL) << 17, - - /** @DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2: M5: versioned render storage. */ - DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2 = (1UL) << 18, - - /** @DRM_ASAHI_FEATURE_SPARSE_TILES: M5: DRM_ASAHI_BIND_SPARSE_TILE. */ - DRM_ASAHI_FEATURE_SPARSE_TILES = (1UL) << 19, - - /** @DRM_ASAHI_FEATURE_COMPUTE_MATRIX: M5: compute matrix instructions. */ - DRM_ASAHI_FEATURE_COMPUTE_MATRIX = (1UL) << 20, - - /** @DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: M5: task and mesh storage. */ - DRM_ASAHI_FEATURE_TASK_MESH_STORAGE = (1UL) << 21, - - /** @DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES: M5: per-render shader writes. */ - DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES = (1UL) << 22, - - /** - * @DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION: M5: compressed - * multisample depth. - */ - DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION = (1UL) << 23, - - /** - * @DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS: M5: a compute prefix for a - * render's tiling half. - */ - DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS = (1UL) << 24, - - /** - * @DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: M5: compressed - * multisample stencil. - */ - DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION = (1UL) << 26, - - /** @DRM_ASAHI_FEATURE_COMPUTE_RESOURCES: M5: per-dispatch storage. */ - DRM_ASAHI_FEATURE_COMPUTE_RESOURCES = (1UL) << 28, - - /** - * @DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS: SUBMIT accepts one - * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs - * include the earlier points of that timeline without adding GPU waits. - */ - DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -445,10 +371,7 @@ enum drm_asahi_feature { * The driver publishes an accepted job's failure before signaling its fence. * Thus userspace must query after a successful DRM syncobj wait/poll before * treating that fence as successful execution. Syncobj signal alone is not - * an execution-success indication. A job cancelled (ECANCELED) because - * another VM's recovery discarded it, an input did not signal in time, or - * its queue was closed, is not a failure of this VM: its fence carries the - * error, error stays 0, and its output is undefined. + * an execution-success indication. * The feature bit is the availability contract; older kernels leave it zero. * Kernel errors for invalid requests: EINVAL flags/pad/size, ENOENT foreign or * missing vm_id, EFAULT inaccessible pointer. No output on invalid request. @@ -703,12 +626,6 @@ enum drm_asahi_bind_flags { * It is intended as a mechanism to accelerate sparse. */ DRM_ASAHI_BIND_SINGLE_PAGE = (1L << 3), - - /** - * @DRM_ASAHI_BIND_SPARSE_TILE: Reserved for the M5 runtime - * (DRM_ASAHI_FEATURE_SPARSE_TILES). This kernel rejects it. - */ - DRM_ASAHI_BIND_SPARSE_TILE = (1L << 4), }; /** @@ -870,24 +787,6 @@ enum drm_asahi_cmd_type { * Followed by (possibly multiple) @drm_asahi_attachment payloads. */ DRM_ASAHI_SET_COMPUTE_ATTACHMENTS = 4, - - /** - * @DRM_ASAHI_SET_RENDER_RESOURCES: Reserved for the M5 runtime - * (DRM_ASAHI_FEATURE_RENDER_RESOURCES). This kernel rejects it. - */ - DRM_ASAHI_SET_RENDER_RESOURCES = 5, - - /** - * @DRM_ASAHI_SET_COMPUTE_RESOURCES: Reserved for the M5 runtime - * (DRM_ASAHI_FEATURE_COMPUTE_RESOURCES). This kernel rejects it. - */ - DRM_ASAHI_SET_COMPUTE_RESOURCES = 6, - - /** - * @DRM_ASAHI_SET_RENDER_RESOURCES_V2: Reserved for the M5 runtime - * (DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2). This kernel rejects it. - */ - DRM_ASAHI_SET_RENDER_RESOURCES_V2 = 7, }; /** @@ -907,8 +806,7 @@ enum drm_asahi_priority { /** * @DRM_ASAHI_PRIORITY_HIGH: High priority queue. * - * Requires CAP_SYS_NICE or DRM master; otherwise queue creation fails - * with -EACCES. + * Reserved for future extension. */ DRM_ASAHI_PRIORITY_HIGH = 2, @@ -982,16 +880,6 @@ enum drm_asahi_sync_type { /** @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ: Timeline sync object */ DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ = 1, - - /** - * @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE: Output-only queue - * timeline anchor, requiring CUMULATIVE_SIGNALS and a nonzero point. - * At most one is allowed. All outputs receive completion of this submit - * and the earlier anchor points. - * Other outputs must not alias the anchor. No execution dependency is - * added. Calls updating this private anchor must be externally serialized. - */ - DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE = 2, }; /** @@ -1726,17 +1614,6 @@ enum drm_asahi_compute_flags { DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF = (1L << 0), DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), - - /** - * @DRM_ASAHI_COMPUTE_AS_BUILD: Acceleration-structure build work, - * including compute-based transfers. Requires BOUNDED_AS_BUILD. - * This is an execution hint, not proof of progress or permission to - * access additional memory. The kernel validates the command normally - * and caps its execution allowance at ten seconds (four extra 2s polls). - * Other commands in the same submission receive the same allowance so - * ordered helpers cannot time out while waiting for the build. - */ - DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2), }; /** diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 56f9f84689804..099e64b3ce5fd 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -30,8 +30,6 @@ #include #include -#include -#include #include #include #include @@ -39,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -67,12 +64,9 @@ #include #include #include -#include #include #include -#include #include -#include #include #include #include @@ -99,18 +93,13 @@ #include #include #include -#include #include #include #include -#include #include #include -#include -#include #include #include -#include #include #include #include @@ -124,8 +113,6 @@ #include #include #include -#include -#include #include #include #include @@ -146,7 +133,6 @@ const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN; const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE; -const size_t RUST_CONST_HELPER_GENLMSG_DEFAULT_SIZE = GENLMSG_DEFAULT_SIZE; const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC; const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL; const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; @@ -157,7 +143,6 @@ const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; -const u32 RUST_CONST_HELPER_APPLE_RTKIT_NEEDS_COREANALYTICS = APPLE_RTKIT_NEEDS_COREANALYTICS; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; @@ -167,17 +152,8 @@ const snd_ctl_elem_iface_t BINDINGS_SNDRV_CTL_ELEM_IFACE_MIXER = SNDRV_CTL_ELEM_ const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; -const u32 BINDINGS_IIO_CHAN_INFO_SCALE = IIO_CHAN_INFO_SCALE; const u32 BINDINGS_IIO_ANGL = IIO_ANGL; const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; -const u32 BINDINGS_IIO_COLORTEMP = IIO_COLORTEMP; -const u32 BINDINGS_IIO_ACCEL = IIO_ACCEL; -const u32 BINDINGS_IIO_ANGL_VEL = IIO_ANGL_VEL; -const u32 BINDINGS_IIO_TIMESTAMP = IIO_TIMESTAMP; -const u32 BINDINGS_IIO_MOD_X = IIO_MOD_X; -const u32 BINDINGS_IIO_MOD_Y = IIO_MOD_Y; -const u32 BINDINGS_IIO_MOD_Z = IIO_MOD_Z; -const u32 BINDINGS_IIO_CHAN_INFO_SAMP_FREQ = IIO_CHAN_INFO_SAMP_FREQ; const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; diff --git a/rust/helpers/mm.c b/rust/helpers/mm.c index d9933c4595682..b5540997bd20d 100644 --- a/rust/helpers/mm.c +++ b/rust/helpers/mm.c @@ -23,18 +23,6 @@ __rust_helper bool rust_helper_mmget_not_zero(struct mm_struct *mm) return mmget_not_zero(mm); } -__rust_helper void rust_helper_add_mm_counter(struct mm_struct *mm, int member, - long value) -{ - add_mm_counter(mm, member, value); -} - -__rust_helper unsigned long rust_helper_get_mm_counter_sum(struct mm_struct *mm, - int member) -{ - return get_mm_counter_sum(mm, member); -} - __rust_helper void rust_helper_mmap_read_lock(struct mm_struct *mm) { mmap_read_lock(mm); diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 3b76d329688db..4d6674f245049 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -192,7 +192,7 @@ impl UnregisteredDevice { dumb_create: T::Object::ALLOC_OPS.dumb_create, dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, - show_fdinfo: Some(drm::File::::show_fdinfo_callback), + show_fdinfo: None, fbdev_probe: None, major: T::INFO.major, @@ -207,8 +207,7 @@ impl UnregisteredDevice { fops: &Self::GEM_FOPS, }; - const GEM_FOPS: bindings::file_operations = - drm::gem::create_fops(crate::module::this_module::().as_ptr()); + const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(T::MODULE); /// Create a new `UnregisteredDevice` for a `drm::Driver`. /// @@ -252,7 +251,7 @@ impl UnregisteredDevice { // SAFETY: // - `raw_data` is a valid pointer to uninitialized memory. // - `raw_data` will not move until it is dropped. - unsafe { pin_init::raw_try_init(raw_data, data) }.inspect_err(|_| { + unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| { // SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the // refcount must be non-zero. unsafe { bindings::drm_dev_put(drm_dev) }; @@ -301,26 +300,6 @@ impl Device { self.dev.get() } - /// Reports that device execution is unavailable and the recovery method is unknown. - /// - /// May sleep: call in task context outside fence signalling and driver locks. - /// A registration guard keeps the parent bound throughout event delivery; a device - /// still being probed or already unplugged cannot deliver the event. - pub fn report_wedged(&self) -> Result { - // SAFETY: Every device context satisfies the weaker Normal context invariants. - let normal = unsafe { self.assume_ctx::() }; - let _registration = normal.registration_guard().ok_or(ENODEV)?; - let raw = self.as_raw(); - // SAFETY: The referenced device and its registered primary node remain live under - // the registration guard. A probe failure has already been rejected above. - if unsafe { (*raw).primary.is_null() } { - return Err(ENODEV); - } - // SAFETY: The device/node are live, 0 asks DRM to report an unknown recovery method, - // and a NULL task means no userspace task is attributed as guilty. - crate::error::to_result(unsafe { bindings::drm_dev_wedged_event(raw, 0, ptr::null_mut()) }) - } - /// # Safety /// /// `ptr` must be a valid pointer to a `struct device` embedded in `Self`. diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 81f0a90e4a517..8a56f00203f56 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -12,23 +12,6 @@ use crate::{ }; use core::marker::PhantomData; -/// Memory a client reports in its fdinfo, in bytes, as `drm--system` lines. -/// -/// See `Documentation/gpu/drm-usage-stats.rst`. `None` leaves an optional key out. -#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] -pub struct MemoryStats { - /// Memory shared with other clients (`drm-shared-*`, also part of `drm-total-*`). - pub shared: u64, - /// Memory private to the client (part of `drm-total-*`). - pub private: u64, - /// Memory currently resident (`drm-resident-*`). - pub resident: Option, - /// Resident memory userspace marked purgeable (`drm-purgeable-*`). - pub purgeable: Option, - /// Memory in use by active submissions (`drm-active-*`). - pub active: Option, -} - /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { /// The parent `Driver` implementation for this `DriverFile`. @@ -39,16 +22,6 @@ pub trait DriverFile { /// Get raw drm_file pointer fn as_raw(&self) -> *mut bindings::drm_file; - - /// Memory statistics for this client's fdinfo. The default, `None`, prints no memory keys. - fn memory_stats(&self) -> Option { - None - } - - /// Emit completed per-engine busy time in nanoseconds for this client's fdinfo. - /// Engine names must be stable, unique and contain no whitespace or colons. - /// The default emits no engine keys. See `Documentation/gpu/drm-usage-stats.rst`. - fn engine_stats(&self, _emit: impl FnMut(&crate::str::CStr, u64)) {} } /// An open DRM File. @@ -75,12 +48,6 @@ impl File { self.0.get() } - /// Whether this file holds the device's DRM master (the compositor or display server). - pub fn is_current_master(&self) -> bool { - // SAFETY: By the type invariants of `Self`, `self.as_raw()` is a valid open `drm_file`. - unsafe { bindings::drm_is_current_master(self.as_raw()) } - } - /// Tag recent feeder work for temporary capacity hints during syncobj waits. /// User scheduling attributes and CPU affinity remain unchanged. pub fn set_syncobj_wait_hint(&self, util_min: u32, duration_ms: u32) { @@ -132,59 +99,6 @@ impl File { 0 } - /// The `show_fdinfo` callback of a `struct drm_driver`. - pub(crate) extern "C" fn show_fdinfo_callback( - printer: *mut bindings::drm_printer, - raw_file: *mut bindings::drm_file, - ) { - // SAFETY: `drm_show_fdinfo()` passes an open `struct drm_file` of this driver, whose - // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the - // file, so `postclose_callback()` cannot run concurrently. - let file = unsafe { File::::from_raw(raw_file) }; - file.inner().engine_stats(|name, ns| { - // SAFETY: The DRM core owns printer throughout this callback; the format and - // name are NUL-terminated, and the vararg types match %s and %llu. - unsafe { - bindings::drm_printf( - printer, - c"drm-engine-%s:\t%llu ns\n".as_char_ptr(), - name.as_char_ptr(), - ns as core::ffi::c_ulonglong, - ) - }; - }); - let Some(stats) = file.inner().memory_stats() else { - return; - }; - - let mut supported = 0; - let mut raw = bindings::drm_memory_stats { - shared: stats.shared, - private: stats.private, - resident: 0, - purgeable: 0, - active: 0, - }; - if let Some(resident) = stats.resident { - raw.resident = resident; - supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_RESIDENT; - } - if let Some(purgeable) = stats.purgeable { - raw.purgeable = purgeable; - supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_PURGEABLE; - } - if let Some(active) = stats.active { - raw.active = active; - supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_ACTIVE; - } - - // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` - // and the region name outlive the call. - unsafe { - bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_char_ptr()) - }; - } - /// The postclose callback of a `struct drm_file`. pub(crate) extern "C" fn postclose_callback( _raw_dev: *mut bindings::drm_device, diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 83eca852139be..fb7e5c38e5483 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -82,12 +82,6 @@ pub struct ObjectConfig<'a, T: DriverObject> { /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. pub reclaimable_cpu_mappings: bool, - /// Fail backing population when it would need direct reclaim. - /// - /// Background reclaim and ordinary allocation placement are preserved. - /// Intended for large, fallible pin requests, not essential kernel backing. - pub pages_no_reclaim: bool, - /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. @@ -100,13 +94,13 @@ impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { Self { map_wc: false, reclaimable_cpu_mappings: false, - pages_no_reclaim: false, parent_resv_obj: None, } } } -const SGT_RECLAIMING: usize = 1 << (usize::BITS - 1); +const SGT_PERMANENT: usize = 1 << (usize::BITS - 1); +const SGT_RECLAIMING: usize = 1 << (usize::BITS - 2); const SGT_USERS: usize = SGT_RECLAIMING - 1; /// A shmem-backed GEM object. @@ -122,12 +116,12 @@ pub struct Object { obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, - /// One cleanup registration for the C SG-table cache; never holds a table borrow. - sgt_cleanup: ManuallyDrop>>>, + /// Devres object for unmapping any SGTable on driver-unbind. + sgt_res: ManuallyDrop>>>, #[pin] - /// Serializes cleanup registration and table acquisition. + /// Lock for protecting initialization of `sgt_res`. sgt_lock: Mutex<()>, - /// Counted SG leases exclude backing eviction; the high bit excludes new leases. + /// Owned SGT readers, or permanent borrowed access, exclude backing eviction. sgt_access: AtomicUsize, #[pin] inner: T, @@ -195,13 +189,13 @@ impl Object { // `Object` let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); - // We need to drop `sgt_cleanup` first, since doing so requires that the GEM object is still + // We need to drop `sgt_res` first, since doing so requires that the GEM object is still // alive. // SAFETY: // - We verified above that `this` is valid. // - We are in free_callback, guaranteeing we have exclusive access to `this` and that - // `sgt_cleanup` will not be used after dropping it here. - unsafe { ManuallyDrop::drop(&mut (*this).sgt_cleanup) }; + // `sgt_res` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; // SAFETY: // - We're in free_callback - so this function is safe to call. @@ -279,59 +273,67 @@ impl Object { self.make_vmap() } - /// Acquire a revocable lease on this object's DMA scatter-gather table. + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. /// - /// Keep the lease for as long as any device mapping or copied DMA address uses the backing. - /// The table borrow cannot outlive the lease. The device must be this object's parent. - /// This may acquire the DMA reservation; call before locking it for a VM_BIND transaction. - pub fn sg_table<'a>(&'a self, dev: &'a device::Device) -> Result> { + /// This will pin the object in memory. It is expected that `dev` should be a pointer to the + /// same [`device::Device`] which `self` belongs to, otherwise this function will return + /// `Err(EINVAL)`. + pub fn sg_table<'a>( + &'a self, + dev: &'a device::Device, + ) -> Result<&'a scatterlist::SGTable> { let parent = self.dev().as_ref(); if dev.as_raw() != parent.as_ref().as_raw() { return Err(EINVAL); } - // Release and purge set the reclaim bit only while they hold this object's reservation. - // Wait for that holder instead of failing a concurrent bind with a transient EBUSY. - while let Err(e) = self.acquire_sgt_access() { - if e != EBUSY { - return Err(e); - } - drop(DmaResvGuard::new(self)); - } - // Every failure after incrementing the count drops this lease. - let lease = SGLease { obj: self.into() }; - - // SAFETY: Import attachment ownership is immutable after publication. - let sgt = if !unsafe { (*self.as_raw()).import_attach }.is_null() { - // Imports belong to the exporter. Never register them for Rust devres cleanup, - // rebuild their tables, or release their backing. - // SAFETY: The attachment table stays valid until this object is freed. + // SAFETY: `import_attach` is set before an imported object is published and does not + // change until the object is freed. + if !unsafe { (*self.as_raw()).import_attach }.is_null() { + // The SG table of an imported object is the dma-buf attachment's mapping, which the + // exporter made for this device at import time. It belongs to the attachment, and + // `drm_gem_shmem_release()` hands it back to the exporter through + // `drm_prime_gem_destroy()`. It must not be placed in `sgt_res`, whose `SGTableMap` + // would unmap and free it as if these helpers had built it. + // + // SAFETY: `sgt` of an imported object is set before the object is published and stays + // valid and unmodified until the object is freed, which `'a` cannot outlive. let sgt = unsafe { (*self.as_raw_shmem()).sgt }; if sgt.is_null() { return Err(EINVAL); } - sgt - } else { - // Register cleanup once, but cache the actual table only in C's shmem->sgt. - // Reclaim clears that pointer. No Rust reference to it survives without a lease. + // SAFETY: See above. + return Ok(unsafe { scatterlist::SGTable::from_raw(sgt) }); + } + + // The borrowed reference can outlive any lock, so pin the backing permanently. + self.acquire_sgt_access(true)?; + + let sgt_res = 'out: { + // Fast path: sgt_res is already initialized + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; + } + + // Slow path: Grab the lock and see if we need to initialize sgt_res. let _guard = self.sgt_lock.lock(); - if self.sgt_cleanup.as_ref().is_none() { - self.sgt_cleanup.populate(Devres::new( - dev, - Ok::<_, Error>(SGTableCleanup { obj: self.into() }), - )?); + + // If someone initialized it while we were waiting, we can exit early. + if let Some(sgt_res) = self.sgt_res.as_ref() { + break 'out sgt_res; } - self.sgt_cleanup.as_ref().ok_or(EIO)?.access(dev)?; - // SAFETY: The live object and lease exclude backing release. The helper locks - // the reservation and either returns the cached table or rebuilds it. - from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.as_raw_shmem()) })? + + // If not, finish initializing and return. `populate()` cannot return false, as + // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. + self.sgt_res + .populate(Devres::new(dev, SGTableMap::new(self))?); + + // SAFETY: We just populated sgt_res above. + unsafe { self.sgt_res.as_ref().unwrap_unchecked() } }; - Ok(SGTable { - // SAFETY: The lease excludes release; the Bound device excludes devres revocation. - table: unsafe { scatterlist::SGTable::from_raw(sgt) }, - lease, - }) + Ok(sgt_res.access(dev)?) } /// Create a new shmem-backed DRM object of the given size. @@ -347,7 +349,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), - sgt_cleanup: ManuallyDrop::new(SetOnce::new()), + sgt_res: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -358,10 +360,6 @@ impl Object { // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; - // SAFETY: The object is initialized but not published. The C initializer - // reads this policy when setting up the backing mapping's GFP mask. - unsafe { (*new.as_raw_shmem()).pages_no_reclaim = config.pages_no_reclaim }; - // SAFETY: The arguments are all valid via the type invariants. to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; @@ -405,7 +403,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: None, - sgt_cleanup: ManuallyDrop::new(SetOnce::new()), + sgt_res: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -431,18 +429,23 @@ impl Object { } } - fn acquire_sgt_access(&self) -> Result { + fn acquire_sgt_access(&self, permanent: bool) -> Result { let mut state = self.sgt_access.load(Ordering::Acquire); loop { if state & SGT_RECLAIMING != 0 { return Err(EBUSY); } - if state & SGT_USERS == SGT_USERS { - return Err(EOVERFLOW); - } + let next = if permanent { + state | SGT_PERMANENT + } else { + if state & SGT_USERS == SGT_USERS { + return Err(EOVERFLOW); + } + state + 1 + }; match self.sgt_access.compare_exchange_weak( state, - state + 1, + next, Ordering::AcqRel, Ordering::Acquire, ) { @@ -463,34 +466,11 @@ impl Object { unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } } - /// Whether the object still has a cached device SG table. - /// - /// # Safety - /// - /// The caller holds the object's DMA reservation lock. - pub unsafe fn has_sg_table_locked(&self) -> bool { - // SAFETY: Reservation ownership serializes the C table pointer. - !unsafe { (*self.as_raw_shmem()).sgt }.is_null() - } - - /// Whether backing without a Rust SG lease can be reclaimed or purged. - /// - /// Also accepts backing whose SG table was already released by idle reclaim. - /// - /// # Safety - /// - /// The caller holds the object's DMA reservation lock. - pub unsafe fn unleased_pages_releasable_locked(&self) -> bool { - self.sgt_access.load(Ordering::Acquire) == 0 && - // SAFETY: The caller establishes the reservation-lock contract. - unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } - } - /// Release idle backing without discarding contents. A later SG table may have different DMA /// addresses. /// - /// Refused with `EBUSY` while any SG lease remains. The cached C table is cleared on - /// success and the next [`Self::sg_table`] rebuilds it before exposing DMA addresses. + /// Refused with `EBUSY` once [`Self::sg_table`] has handed out a borrowed table, since that + /// borrow may outlive any lock. /// /// # Safety /// @@ -536,9 +516,7 @@ impl Object { /// invalidated every device mapping, and no Rust SG lease remains. pub unsafe fn purge_locked(&self) -> Result { // SAFETY: The caller's contract; the exclusive state below excludes new leases. - if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } - || !unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } - { + if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } { return Err(EBUSY); } self.sgt_access @@ -728,72 +706,63 @@ where { } -/// An owned backing lease, suitable for retaining copied DMA segments. +/// A reference to a GEM object that is known to have a mapped [`SGTable`]. /// -/// Dropping the last lease permits reclaim after device translations have been invalidated. -/// This does not itself provide access to the table: table access also requires a Bound device. -#[must_use = "retain until all device uses of the backing have ended"] -pub struct SGLease { - obj: ARef>, -} - -impl Drop for SGLease { - fn drop(&mut self) { - let old = self.obj.sgt_access.fetch_sub(1, Ordering::Release); - debug_assert!(old & SGT_USERS != 0 && old & SGT_RECLAIMING == 0); - } -} - -/// A leased SG-table borrow. References obtained from it cannot outlive the lease. -#[must_use = "retain while using the SG table or transfer into an owned lease"] -pub struct SGTable<'a, T: DriverObject> { - table: &'a scatterlist::SGTable, - lease: SGLease, +/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables +/// on GEM shmem objects are revoked on driver-unbind. +/// +/// # Invariants +/// +/// - `self.obj` always points to a valid GEM object. +/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an +/// [`SGTable`]. +/// +/// [`SGTable`]: scatterlist::SGTable +pub struct SGTableMap { + obj: NonNull>, } -impl Deref for SGTable<'_, T> { +impl Deref for SGTableMap { type Target = scatterlist::SGTable; fn deref(&self) -> &Self::Target { - self.table - } -} - -impl SGTable<'_, T> { - /// Retain the backing after copying the DMA addresses, without retaining a table borrow. - pub fn into_lease(self) -> SGLease { - self.lease + // SAFETY: + // - The NonNull is guaranteed to be valid via our type invariants. + // - The sgt field is guaranteed to be initialized and valid via our type invariants. + unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } } } -/// Device-unbind cleanup registration for locally allocated SG tables. -/// -/// This token owns no table reference: reclaim may clear shmem->sgt and a later lease may -/// rebuild it. It is dropped before the GEM object, and is never constructed for imports. -struct SGTableCleanup { - obj: NonNull>, -} - -impl Drop for SGTableCleanup { +impl Drop for SGTableMap { fn drop(&mut self) { - // SAFETY: The registration is dropped before its GEM object. + // SAFETY: `obj` is always valid via our type invariants let obj = unsafe { self.obj.as_ref() }; let _lock = DmaResvGuard::new(obj); - // SAFETY: Reservation ownership serializes shmem->sgt. Imports never get this token. - // Reclaim may already have freed the table; the C free helper requires a non-null one. - unsafe { - if !(*obj.as_raw_shmem()).sgt.is_null() { - bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()); - } - } + // SAFETY: We acquired the lock needed for calling this function above + unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; } } -// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. -unsafe impl Send for SGTableCleanup {} -// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. -unsafe impl Sync for SGTableCleanup {} +impl SGTableMap { + fn new(obj: &Object) -> impl Init { + // INVARIANT: + // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, + // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. + // SAFETY: + // - `obj` is fully initialized, making this function safe to call. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; + + Ok(Self { obj: obj.into() }) + } +} + +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Send for SGTableMap {} +// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object +// it points to is guaranteed to be thread-safe. +unsafe impl Sync for SGTableMap {} #[kunit_tests(rust_drm_gem_shmem)] mod tests { @@ -870,14 +839,6 @@ mod tests { let data = try_pin_init!(KunitData {}); let reg = faux::Registration::new(c"Kunit", None)?; let fdev = reg.as_ref(); - let dev: &device::Device = fdev.as_ref(); - // SAFETY: This new faux device is private to the fixture. Use direct DMA with a - // 64-bit mask, as the C shmem KUnit fixture does, before publishing any mappings. - unsafe { - let raw = dev.as_raw(); - (*raw).coherent_dma_mask = u64::MAX; - (*raw).dma_mask = &raw mut (*raw).coherent_dma_mask; - } let drm = UnregisteredDevice::new(fdev, data)?; Ok((reg, drm)) @@ -924,225 +885,19 @@ mod tests { // Ensure the ordering in memory is correct let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); for (offset, expected) in (0x20..=0x23).zip(expected) { - assert_eq!(vmap.try_read8(offset).unwrap(), expected); + assert_eq!(vmap.read8(offset), expected); } Ok(()) } - #[test] - fn sg_lease_counting() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - let first = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); - let second = obj.sg_table(dev.as_ref().as_ref())?; - assert_eq!(obj.sgt_access.load(Ordering::Acquire), 2); - drop(first); - assert_eq!(obj.sgt_access.load(Ordering::Acquire), 1); - drop(second); - assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); - Ok(()) - } - - #[test] - fn release_idle_without_mappings() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - drop(obj.sg_table(dev.as_ref().as_ref())?); - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held, and no device mappings or SG leases exist in this test. - unsafe { obj.release_idle_pages_locked()? }; - // SAFETY: Reservation ownership serializes these fields. - let empty = unsafe { - ( - (*obj.as_raw_shmem()).sgt.is_null(), - (*obj.as_raw_shmem()).pages.is_null(), - ) - }; - assert!(empty.0); - assert!(empty.1); - // SAFETY: Still no device mappings or leases; an already idle release succeeds too. - unsafe { obj.release_idle_pages_locked()? }; - Ok(()) - } - - #[test] - fn release_refused_while_leased() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); - let bind = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; no actual device mappings were installed. - assert!(!unsafe { obj.sole_sg_lease_releasable_locked() }); - // SAFETY: Reservation held; no actual device mappings were installed. - assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); - drop(bind); - // SAFETY: Reservation held; the mapping lease is now the sole owner. - assert!(unsafe { obj.sole_sg_lease_releasable_locked() }); - // SAFETY: Reservation held; the remaining lease must still exclude release. - assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); - drop(mapping); - // SAFETY: Reservation held; no device mapping or SG lease remains. - unsafe { obj.release_idle_pages_locked()? }; - Ok(()) - } - - #[test] - fn purge_madvised_after_last_lease() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); - assert!(obj.madvise(1)); - { - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; no real device access. - assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); - drop(mapping); - // SAFETY: Reservation held; all device uses and SG leases have ended. - unsafe { obj.purge_locked()? }; - // SAFETY: Reservation ownership serializes the backing fields and advice. - let (empty_sgt, empty_pages, madv) = unsafe { - ( - (*obj.as_raw_shmem()).sgt.is_null(), - (*obj.as_raw_shmem()).pages.is_null(), - obj.madv_locked(), - ) - }; - assert!(empty_sgt); - assert!(empty_pages); - assert_eq!(madv, -1); - } - assert!(!obj.madvise(0)); - assert!(obj.sg_table(dev.as_ref().as_ref()).is_err()); - assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); - Ok(()) - } - - #[test] - fn sg_rebuild_after_release() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - obj.vmap::<4>()?.as_view().write32(0x12345678, 0); - for _ in 0..3 { - let table = obj.sg_table(dev.as_ref().as_ref())?; - assert!(table.iter().count() > 0); - drop(table); - { - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; all leases have dropped, with no device mappings. - unsafe { obj.release_idle_pages_locked()? }; - // SAFETY: Reservation ownership serializes the table pointer. - assert!(unsafe { (*obj.as_raw_shmem()).sgt.is_null() }); - } - assert_eq!(obj.vmap::<4>()?.as_view().read32(0), 0x12345678); - } - // Drop with a rebuilt table exercises the surviving devres cleanup registration too. - drop(obj.sg_table(dev.as_ref().as_ref())?); - Ok(()) - } - - #[test] - fn purge_after_idle_release() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - drop(obj.sg_table(dev.as_ref().as_ref())?); - { - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; no device access or SG leases remain. - unsafe { obj.release_idle_pages_locked()? }; - } - assert!(obj.madvise(1)); - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; no device access or SG leases remain. - assert!(unsafe { obj.unleased_pages_releasable_locked() }); - // SAFETY: Reservation held; no device access or SG lease remains. - unsafe { obj.purge_locked()? }; - // SAFETY: Reservation ownership serializes the advice and table pointer. - let (madv, empty) = unsafe { (obj.madv_locked(), (*obj.as_raw_shmem()).sgt.is_null()) }; - assert_eq!(madv, -1); - assert!(empty); - Ok(()) - } - - #[test] - fn purge_refuses_kernel_vmap() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - drop(obj.sg_table(dev.as_ref().as_ref())?); - let vmap = obj.vmap::<4>()?; - assert!(obj.madvise(1)); - { - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; the vmap must prevent the backing from being freed. - assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); - // SAFETY: Reservation held; the vmap must exclude idle release too. - assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); - } - drop(vmap); - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held; no device, vmap or SG users remain. - unsafe { obj.purge_locked()? }; - Ok(()) - } - - #[test] - fn sg_reclaim_excludes_new_leases() -> Result { - let (_dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - obj.sgt_access.store(SGT_RECLAIMING, Ordering::Release); - assert_eq!(obj.acquire_sgt_access(), Err(EBUSY)); - assert_eq!(obj.sgt_access.load(Ordering::Acquire), SGT_RECLAIMING); - obj.sgt_access.store(SGT_USERS, Ordering::Release); - assert_eq!(obj.acquire_sgt_access(), Err(EOVERFLOW)); - obj.sgt_access.store(0, Ordering::Release); - Ok(()) - } - - #[test] - fn sg_import_attachment_is_not_owned() -> Result { - let (dev, drm) = create_drm_dev()?; - let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; - let mut table = MaybeUninit::::zeroed(); - let mut attachment = MaybeUninit::::zeroed(); - // SAFETY: Private fixture; neither mock is dereferenced by the import path. Both stay - // live until the lease drops and are detached before the real object destructor runs. - unsafe { - (*obj.as_raw()).import_attach = attachment.as_mut_ptr(); - (*obj.as_raw_shmem()).sgt = table.as_mut_ptr(); - } - let result = obj.sg_table(dev.as_ref().as_ref()).map(|lease| { - let no_cleanup = obj.sgt_cleanup.as_ref().is_none(); - let _guard = DmaResvGuard::new(&obj); - // SAFETY: Reservation held. Capture results before restoring the fixture, so a - // failed assertion cannot leave a mock attachment for the GEM destructor. - unsafe { - let held = obj.release_idle_pages_locked(); - drop(lease); - ( - no_cleanup, - held, - obj.release_idle_pages_locked(), - obj.purge_locked(), - (*obj.as_raw_shmem()).sgt == table.as_mut_ptr(), - ) - } - }); - // SAFETY: The mock table borrow has ended; restore the private fixture for normal free. - unsafe { - (*obj.as_raw()).import_attach = ptr::null_mut(); - (*obj.as_raw_shmem()).sgt = ptr::null_mut(); - } - let (no_cleanup, held, unleased, purge, unchanged) = result?; - assert!(no_cleanup); - assert_eq!(held, Err(EBUSY)); - assert_eq!(unleased, Err(EINVAL)); - assert_eq!(purge, Err(EBUSY)); - assert!(unchanged); - Ok(()) - } + // TODO: I would love to actually test the success paths of sg_table(), but that would require + // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave + // that for someone else. - // A wrong device must fail before registering cleanup or taking any SG lease. + // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it + // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device + // bound to it. #[test] fn fail_sg_table_on_wrong_dev() -> Result { let (_dev, drm) = create_drm_dev()?; @@ -1153,10 +908,10 @@ mod tests { assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - // A failed lookup must not bind the cleanup registration to the wrong device. + // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + // TODO: Someday, we should test that creating an sg_table here still succeeds. Ok(()) } diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index e43d5ae131c85..4764d7b68f2a7 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -151,59 +151,6 @@ impl Mm { } } -/// A resident-set counter of an address space (`MM_*PAGES` in `struct mm_struct`). -/// -/// The counters make up the RSS that `/proc//status`, `oom_badness()` and the OOM report -/// show for a process. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum RssCounter { - /// Resident file-backed pages (`MM_FILEPAGES`). - File, - /// Resident anonymous pages (`MM_ANONPAGES`). - Anon, - /// Swapped-out anonymous pages (`MM_SWAPENTS`). - Swap, - /// Resident shared-memory pages (`MM_SHMEMPAGES`). - Shmem, -} - -impl RssCounter { - fn as_raw(self) -> core::ffi::c_int { - let member = match self { - Self::File => bindings::MM_FILEPAGES, - Self::Anon => bindings::MM_ANONPAGES, - Self::Swap => bindings::MM_SWAPENTS, - Self::Shmem => bindings::MM_SHMEMPAGES, - }; - member as core::ffi::c_int - } -} - -// The RSS counters live as long as the `mm_struct` itself (until `__mmdrop()`), so these -// methods are safe even if `mm_users` is zero. -impl Mm { - /// Adds `pages` (which may be negative) to one of the resident-set counters. - /// - /// This is how a driver attributes pages it holds on behalf of a process, but which are not - /// mapped into it, to that process. Every page added must be subtracted again before the - /// last `mmgrab` reference is dropped: `__mmdrop()` reports a non-zero counter as a bad - /// rss-counter state. - #[inline] - pub fn add_rss_counter(&self, counter: RssCounter, pages: isize) { - // SAFETY: The pointer is valid since self is a reference, and the counters are initialized - // for as long as the `mm_struct` is. - unsafe { bindings::add_mm_counter(self.as_raw(), counter.as_raw(), pages as _) }; - } - - /// Returns the exact current value of one of the resident-set counters. - #[inline] - pub fn rss_counter(&self, counter: RssCounter) -> usize { - // SAFETY: The pointer is valid since self is a reference, and the counters are initialized - // for as long as the `mm_struct` is. - unsafe { bindings::get_mm_counter_sum(self.as_raw(), counter.as_raw()) as usize } - } -} - // These methods require `mm_users` to be non-zero. impl MmWithUser { /// Obtain a reference from a raw pointer. From a8f092b08ba07ebf988d48418001565203f93d26 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 19:13:02 -0400 Subject: [PATCH 09/37] drm/asahi: sync the G17 driver with aurora-silicon at d3cf19be Bring drivers/gpu/drm/asahi up to the aurora-silicon/linux state at d3cf19be, the base of the G17 performance series that follows, merged three-way on top of integration/aurora with the 7.2.10 merge point 44d5afc7 as base. integration/aurora's own work is kept: the G15 backend, the M3 T8122 runtime, UAT adoption, deferred-close binding sequences (#214) and the 7.2.11 shared-code adaptations. The G16 (T8132) backend is not carried. This carries the G17 work merged on the aurora side since 7.2.10, including the T6040 (M4 Pro) and 25G83 firmware support and the render replay/recovery series by Jonathan Goyvaerts, the watchdog, replay and backing-population changes by Ryan Murray, the tracked VM lifetime work by Eryk Wieliczko, and the G17 queue, capacity and timestamp changes by DjDeveloperr (aurora-silicon/linux #382, #391, #397, #400). Adaptations to 7.2.11: - module parameters return references; THIS_MODULE replaces kernel::module::this_module; a local raw_try_init replaces the newer pin-init helper; of_device_table! takes the MODULE_OF_TABLE name. - regs.rs keeps G17's sgx window as Devres, as 7.2 has no DevresIoMem. - The shared feature flags keep G17's opt-out of soft faults. - G15 and the M3 runtime use the shared timestamp::Buffer and the reserved-table aware root walk; omacom's G15 vm_closed hook becomes vm_destroyed, next to the G17 vm_closed. - Crate and helper pieces the driver needs: the shmem SG lease and pages_no_reclaim (drm_gem_shmem_helper), DRM file memory stats, fdinfo and is_current_master, and the mm RSS counter. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/asahi_core.rs | 14 + drivers/gpu/drm/asahi/driver.rs | 13 +- drivers/gpu/drm/asahi/file.rs | 153 ++- drivers/gpu/drm/asahi/g15/command.rs | 3 +- drivers/gpu/drm/asahi/g15/mod.rs | 6 +- drivers/gpu/drm/asahi/g15/uat.rs | 2 +- drivers/gpu/drm/asahi/g17/buffer.rs | 64 +- drivers/gpu/drm/asahi/g17/buffer/target.rs | 61 +- drivers/gpu/drm/asahi/g17/capacity.rs | 88 ++ drivers/gpu/drm/asahi/g17/command.rs | 50 +- drivers/gpu/drm/asahi/g17/completion.rs | 130 ++- drivers/gpu/drm/asahi/g17/context.rs | 45 +- drivers/gpu/drm/asahi/g17/context/identity.rs | 9 + drivers/gpu/drm/asahi/g17/data_segment.rs | 56 ++ drivers/gpu/drm/asahi/g17/dependency.rs | 8 + drivers/gpu/drm/asahi/g17/event.rs | 8 +- drivers/gpu/drm/asahi/g17/feed.rs | 23 +- drivers/gpu/drm/asahi/g17/freelist.rs | 141 ++- drivers/gpu/drm/asahi/g17/fw/buffer.rs | 4 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 2 +- drivers/gpu/drm/asahi/g17/fw/compute.rs | 77 +- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 157 ++- .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 681 +++++++++++++ drivers/gpu/drm/asahi/g17/fw/kick.rs | 16 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 5 +- drivers/gpu/drm/asahi/g17/fw/queue.rs | 196 +++- drivers/gpu/drm/asahi/g17/fw/render.rs | 303 +++++- drivers/gpu/drm/asahi/g17/fw/variant.rs | 133 +++ drivers/gpu/drm/asahi/g17/initdata.rs | 202 ++-- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 238 +++++ drivers/gpu/drm/asahi/g17/job.rs | 71 +- drivers/gpu/drm/asahi/g17/kick.rs | 381 ++++++- drivers/gpu/drm/asahi/g17/mod.rs | 469 +++++++-- drivers/gpu/drm/asahi/g17/object.rs | 112 ++- drivers/gpu/drm/asahi/g17/power.rs | 123 +++ drivers/gpu/drm/asahi/g17/preparation.rs | 153 +++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 474 ++++++++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 456 +++++++-- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 3 + .../drm/asahi/g17/queue/compute/retirement.rs | 180 +++- drivers/gpu/drm/asahi/g17/queue/mod.rs | 36 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 518 +++++++--- .../gpu/drm/asahi/g17/queue/render/memory.rs | 111 ++- .../gpu/drm/asahi/g17/queue/render/parked.rs | 43 +- .../drm/asahi/g17/queue/render/publication.rs | 34 +- .../drm/asahi/g17/queue/render/retirement.rs | 8 +- .../gpu/drm/asahi/g17/queue/render/warm.rs | 3 +- drivers/gpu/drm/asahi/g17/recovery.rs | 35 +- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 84 ++ drivers/gpu/drm/asahi/g17/recovery/slot.rs | 8 +- drivers/gpu/drm/asahi/g17/replay.rs | 109 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 605 +++++++++-- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 102 +- .../drm/asahi/g17/runtime/compute_control.rs | 18 +- .../drm/asahi/g17/runtime/compute_replay.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 19 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 306 +++++- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 7 +- .../drm/asahi/g17/runtime/render/observe.rs | 6 +- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 8 + .../drm/asahi/g17/runtime/render/prepare.rs | 77 +- .../drm/asahi/g17/runtime/render/replay.rs | 148 +-- .../drm/asahi/g17/runtime/render/settle.rs | 16 +- .../drm/asahi/g17/runtime/render/submit.rs | 218 +++- .../drm/asahi/g17/runtime/render/teardown.rs | 13 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 +- drivers/gpu/drm/asahi/g17/teardown.rs | 15 + drivers/gpu/drm/asahi/g17/timeout.rs | 257 ++++- drivers/gpu/drm/asahi/g17/validation.rs | 5 +- drivers/gpu/drm/asahi/gem.rs | 68 +- drivers/gpu/drm/asahi/gpu.rs | 59 +- drivers/gpu/drm/asahi/hw/mod.rs | 2 + drivers/gpu/drm/asahi/hw/t6040.rs | 180 ++++ drivers/gpu/drm/asahi/hw/t8140.rs | 37 +- drivers/gpu/drm/asahi/m3/m3_drm.rs | 5 +- drivers/gpu/drm/asahi/m3/m3_submit.rs | 5 +- drivers/gpu/drm/asahi/mem.rs | 42 +- drivers/gpu/drm/asahi/memacct.rs | 549 ++++++++++ drivers/gpu/drm/asahi/mmu.rs | 943 ++++++++++++++---- drivers/gpu/drm/asahi/mmu/bind.rs | 13 +- drivers/gpu/drm/asahi/mmu/lifetime.rs | 79 +- drivers/gpu/drm/asahi/mmu/m3.rs | 2 +- drivers/gpu/drm/asahi/mmu/ppl.rs | 71 ++ drivers/gpu/drm/asahi/mmu/residency.rs | 69 +- drivers/gpu/drm/asahi/mmu/shared.rs | 4 +- drivers/gpu/drm/asahi/object.rs | 2 +- drivers/gpu/drm/asahi/pgtable.rs | 409 +++++--- drivers/gpu/drm/asahi/queue/common.rs | 2 +- drivers/gpu/drm/asahi/queue/mod.rs | 20 +- drivers/gpu/drm/asahi/regs.rs | 64 +- drivers/gpu/drm/asahi/sysfs.c | 106 +- drivers/gpu/drm/asahi/sysfs.h | 3 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 21 +- drivers/gpu/drm/asahi/timestamp.rs | 185 ++++ drivers/gpu/drm/asahi/util.rs | 14 + drivers/gpu/drm/drm_gem_shmem_helper.c | 20 +- include/drm/drm_gem_shmem_helper.h | 11 +- include/uapi/drm/asahi_drm.h | 34 +- rust/bindings/bindings_helper.h | 1 + rust/helpers/mm.c | 12 + rust/kernel/drm/device.rs | 2 +- rust/kernel/drm/file.rs | 69 ++ rust/kernel/drm/gem/shmem.rs | 487 ++++++--- rust/kernel/mm.rs | 53 + 104 files changed, 10248 insertions(+), 1518 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/capacity.rs create mode 100644 drivers/gpu/drm/asahi/g17/data_segment.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/variant.rs create mode 100644 drivers/gpu/drm/asahi/g17/initdata/v25g83.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery/safety.rs create mode 100644 drivers/gpu/drm/asahi/g17/replay.rs create mode 100644 drivers/gpu/drm/asahi/hw/t6040.rs create mode 100644 drivers/gpu/drm/asahi/memacct.rs create mode 100644 drivers/gpu/drm/asahi/mmu/ppl.rs create mode 100644 drivers/gpu/drm/asahi/timestamp.rs diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 0fbe6e728773d..8b782a6f8ffa7 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -25,6 +25,7 @@ mod gpu; mod hw; mod initdata; mod mem; +mod memacct; mod m3; mod microseq; mod mmu; @@ -35,6 +36,7 @@ mod regs; mod slotalloc; mod stats; mod sysfs_exports; +mod timestamp; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -172,5 +174,17 @@ module! { // permissions: 0, description: "Export AGX firmware stats to sysfs (1 = on, 0 = off)", }, + mem_limit_pct: u32 { + default: 67, + description: "Device-wide GPU memory limit in percent of RAM (0 or >= 100: none)", + }, + client_mem_limit_pct: u32 { + default: 50, + description: "Per-client GPU memory limit in percent of RAM (0: the device limit)", + }, + oom_attribution: u8 { + default: 1, + description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", + }, }, } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 43fa17412815d..9b9b8fc4ccd15 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -103,7 +103,6 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; @@ -153,6 +152,10 @@ kernel::of_device_table!( of::DeviceId::new(c"apple,agx-t8140"), ProbeConfig::G17(&hw::t8140::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t6040"), + ProbeConfig::G17(&hw::t6040::CONFIG) + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -233,6 +236,7 @@ impl platform::Driver for AsahiPlatformDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { debug::update_debug_flags(); + crate::memacct::LEDGER.configure(); dev_info!(pdev.as_ref(), "Probing...\n"); @@ -419,6 +423,11 @@ impl AsahiPlatformDriver { return Err(error); } }; + if let Err(error) = + crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) + { + dev_warn!(pdev.as_ref(), "idle_off_delay_ms not registered: {:?}\n", error); + } Ok(AsahiPlatformDriverData { _device: reg.device().into(), _reg: reg, @@ -433,7 +442,7 @@ impl AsahiPlatformDriver { impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { - // SAFETY: `self.raw_dev` was set in `probe` from the platform + // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index f2f055931d29a..c627bd9c6b8a0 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,6 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, + memacct, + gpu, mmu, queue, util::{ @@ -43,6 +45,7 @@ use kernel::sync::{ Mutex, // }; use kernel::time::NSEC_PER_SEC; +use kernel::types::ScopeGuard; use kernel::uaccess::{ UserPtr, UserSlice, // @@ -68,6 +71,7 @@ struct Vm { user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, + gpu: Arc, } impl Drop for Vm { @@ -89,6 +93,7 @@ impl Drop for Vm { pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); } self.vm.bo_deferred_cleanup(); + self.gpu.vm_closed(&self.vm); return; } // When the user Vm is dropped, verify that no user mappings exists. @@ -218,7 +223,7 @@ impl SyncItem { #[derive(Clone)] pub(crate) enum Object { - TimestampBuffer(Arc), + TimestampBuffer(Arc), } /// State associated with a client. @@ -236,6 +241,8 @@ pub(crate) struct File { /// Largest size this file has asked GET_PARAMS for the global parameters with /// ([`gpu::Gpu::admit_vm`]). params_size: core::sync::atomic::AtomicU64, + /// GPU memory this client created and holds. + mem: Arc, } /// Convenience type alias for our DRM `File` type. @@ -255,9 +262,20 @@ impl drm::file::DriverFile for File { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); let user_range = gpu.user_range(); + let mem = memacct::Client::new()?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) + KBox::pin_init(File::new(id, user_range, mem), GFP_KERNEL) + }) + } + + /// The objects this client created, which `memacct` attributes to it, not every object it + /// has a handle to: an imported or shared buffer is counted once, by its creator. + fn memory_stats(&self) -> Option { + Some(drm::file::MemoryStats { + private: self.mem.committed(), + resident: Some(self.mem.resident()), + ..Default::default() }) } @@ -270,26 +288,37 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new(id: u64, user_range: Range) -> impl PinInit { + fn new( + id: u64, + user_range: Range, + mem: Arc, + ) -> impl PinInit { unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_vms)?; + crate::util::raw_try_init( + raw_vms, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; let raw_queues = addr_of_mut!((*slot).queues); - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ) - .__pinned_init(raw_queues)?; + crate::util::raw_try_init( + raw_queues, + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ), + )?; let raw_objects = addr_of_mut!((*slot).objects); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_objects)?; + crate::util::raw_try_init( + raw_objects, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; (*slot).id = id; (*slot).user_range = user_range; addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); + addr_of_mut!((*slot).mem).write(mem); Ok(()) }) } @@ -341,7 +370,7 @@ impl File { let limits = gpu.queue_limits().ok_or(EINVAL)?; let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; UserSlice::new( - UserPtr::from_addr(data.pointer as _), + UserPtr::from_addr(data.pointer as usize), core::mem::size_of_val(&words), ) .writer() @@ -360,8 +389,11 @@ impl File { return Err(ENODEV); } + let memory_limits = memacct::LEDGER.limits(); let mut params = uapi::drm_asahi_params_global { features: 0, + memory_limit: memory_limits.device, + client_memory_limit: memory_limits.client, vm_start: file.inner().user_range.start, vm_end: file.inner().user_range.end, @@ -383,7 +415,7 @@ impl File { // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { @@ -402,15 +434,18 @@ impl File { if data.pad != 0 || data.size < size as u64 { return Err(EINVAL); } - let user = UserPtr::from_addr(data.pointer as _); + let user = UserPtr::from_addr(data.pointer as usize); let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; if flags != 0 || pad != 0 { return Err(EINVAL); } let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size) - .writer() - .write(&[vm_id, 0, status.reported_error() as u32, 0])?; + UserSlice::new(user, size).writer().write(&[ + vm_id, + 0, + status.reported_error() as u32, + 0, + ])?; Ok(0) } @@ -421,7 +456,7 @@ impl File { return Err(EINVAL); } let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size) + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size) .reader() .read::<[u32; 4]>()?; if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { @@ -492,7 +527,22 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm(kernel_range.clone())?; + let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; + // G16 registers a retained backend owner before returning this VM. Any later + // allocation can fail before file::Vm exists to run its normal close path. + let failed_vm = ScopeGuard::new_with_data( + (vm.clone(), user_range.clone(), kernel_range.clone()), + |(vm, user_range, kernel_range)| { + if gpu.tracked_submission() && vm.status().is_some() && !vm.is_closed() { + vm.close_idle_reclaim(); + if let Err(error) = vm.unmap_user_ranges(user_range, kernel_range) { + pr_err!("VM create: closing unpublished VM failed: {:?}\n", error); + } + vm.bo_deferred_cleanup(); + gpu.vm_closed(&vm); + } + }, + ); let vm_xa = file.inner().vms(); let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -568,10 +618,12 @@ impl File { user_range, kernel_range, _dummy_mapping: dummy_mapping, + gpu: gpu.clone(), }, GFP_KERNEL, )?)?; + drop(failed_vm.dismiss()); data.vm_id = id; Ok(0) @@ -589,7 +641,7 @@ impl File { Err(ENOENT) } else { drop(vm); - reg_data.gpu.vm_closed(); + reg_data.gpu.vm_destroyed(); Ok(0) } } @@ -632,8 +684,24 @@ impl File { None }; - let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; - + let size = gem::checked_object_size(data.size.try_into()?)?; + let commit = memacct::LEDGER.admit(&file.inner().mem, size as u64).inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; + // Large G17 user objects are populated in full by the first VM_BIND. + // Do not multiply direct reclaim/compaction latency by their folio + // count. Userspace can evict its BO cache and retry after ENOMEM. + let pages_no_reclaim = gem::user_pages_no_reclaim( + size, + reg_data.gpu.as_any().is::(), + ); + let gem = gem::new_object( + device, size, data.flags, resv_obj, reclaim_cpu, pages_no_reclaim, commit, + )?; if reg_data.gpu.prepare_new_objects() { gem::prepare_new_object(&gem)?; } @@ -713,11 +781,17 @@ impl File { } if let Some(limit) = reg_data.gpu.bind_batch_limit() { - if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 - || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { + if data.num_binds == 0 + || data.num_binds > limit + || data.userptr == 0 + || data.stride % 8 != 0 + || data.stride as usize > kernel::page::PAGE_SIZE + { + cls_pr_debug!(Errors, "vm_bind: Batch outside the GPU's limits\n"); return Err(EINVAL); } } + if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -728,7 +802,7 @@ impl File { let size = usize::try_from(data.stride)? .checked_mul(usize::try_from(data.num_binds)?) .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as usize), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -746,7 +820,7 @@ impl File { let stride = usize::try_from(data.stride)?; let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as _), size) + UserSlice::new(UserPtr::from_addr(data.userptr as usize), size) .reader() .read_all(&mut bytes, GFP_KERNEL)?; let (vm, user_range, kernel_range) = { @@ -1194,11 +1268,27 @@ impl File { return Err(EINVAL); } - // TODO: Allow with CAP_SYS_NICE - if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { + // REALTIME stays reserved. HIGH needs CAP_SYS_NICE or DRM master, as amdgpu and + // panthor require for raised queue priority; EACCES lets userspace fall back to MEDIUM. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } + // HIGH has only been run on G17; G13 and G14 keep refusing it. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + && !reg_data.gpu.as_any().is::() + { + cls_pr_debug!(Errors, "queue_create: HIGH priority is not supported on this GPU\n"); + return Err(EINVAL); + } + // SAFETY: capable() only reads the calling task's credentials (ioctl process context). + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } + && !file.is_current_master() + { + cls_pr_debug!(Errors, "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n"); + return Err(EACCES); + } let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -1260,6 +1350,9 @@ impl File { } let gpu = ®_data.gpu; + if data.cmdbuf_size as usize > gpu.max_cmdbuf_size() { + return Err(E2BIG); + } if let Some(manager) = gpu.manager() { manager.update_globals(); } @@ -1292,7 +1385,7 @@ impl File { let mut vec = KVec::new(); let copy_commands = |bytes: &mut KVec| { UserSlice::new( - UserPtr::from_addr(data.cmdbuf as _), + UserPtr::from_addr(data.cmdbuf as usize), data.cmdbuf_size as usize, ) .reader() @@ -1369,7 +1462,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = gpu.base_clock_hz() as u64; + let frequency_hz = u64::from(gpu.base_clock_hz()); let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index 45b65d1ddc026..b343aa3fe4e5b 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -360,7 +360,8 @@ impl Timestamps<'_> { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(mapping) = object; + let file::Object::TimestampBuffer(buffer) = object; + let mapping = buffer.firmware_mapping()?; let offset = timestamp.offset as usize; if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 95f04114ba874..48e6afa948c98 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -632,7 +632,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_closed(&self) { + fn vm_destroyed(&self) { vm::closed(self) } @@ -658,7 +658,7 @@ impl gpu::Gpu for Gpu { &self, bo: gem::ObjectRef, range: Range, - ) -> Result { - timestamp::map(self, bo, range) + ) -> Result { + crate::timestamp::Buffer::firmware(timestamp::map(self, bo, range)?) } } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 7ab3a1919eaec..1d63fb83b9fa1 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -99,7 +99,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(pa, count, f) } + unsafe { pgtable::with_root_entries(None, pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index d9b0c887deea0..8f3a03fbb8dba 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -78,7 +78,7 @@ impl BufferIds { self.entries .iter() .position(|entry| entry.free()) - .ok_or(ENOSPC)? as u8 + .ok_or_else(|| super::capacity::refused(super::capacity::Site::ParameterBuffers))? as u8 }; if !self.entries[usize::from(id)].free() { return Err(EIO); @@ -503,7 +503,7 @@ impl MetricsLease { break; } } - let slot = selected.ok_or(ENOSPC)?; + let slot = selected.ok_or_else(|| super::capacity::refused(super::capacity::Site::MetricsIds))?; // Construct the lease before any fallible address/view validation so // every constructor failure returns the selected bit. let mut lease = Self { @@ -556,6 +556,8 @@ pub(crate) struct Manager { buffer_id: u8, generation: u64, shrink: Shrink, + /// A tiling pass initialized this buffer manager (25G83 InitBM published). + initialized: bool, } /// Native tiler-heap shrink policy, matching the heap behaviour seen @@ -657,8 +659,18 @@ impl Manager { buffer_id: args.buffer_id, generation: args.generation, shrink: Shrink::default(), + initialized: false, }) } + + /// Whether a published tiling pass initialized this manager (25G83 InitBM). + pub(crate) fn initialized(&self) -> bool { + self.initialized + } + + pub(crate) fn mark_initialized(&mut self) { + self.initialized = true; + } pub(crate) fn buffer_id(&self) -> u8 { self.buffer_id } @@ -1119,3 +1131,51 @@ fn append_reserved(values: &mut KVec, value: T) { // SAFETY: All callers preflighted this spare entry and initialized it once. unsafe { values.set_len(length + 1) }; } + +impl BufferIds { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "parameter_buffers used={} total={} owned={} referenced={}", + self.entries.iter().filter(|entry| !entry.free()).count(), BUFFER_IDS, + self.entries.iter().filter(|entry| entry.owner).count(), + self.entries.iter().filter(|entry| entry.references != 0).count()) + } +} +impl MetricsIds { + pub(crate) fn used(&self) -> u32 { + self.used.iter().map(|bits| bits.load(Ordering::Acquire).count_ones()).sum::() + - (128 - fw::METRICS_SLOTS as u32) + } +} + +#[kunit_tests(asahi_g17_buffer_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn failed_build_and_fault_retirement_return_parameter_buffers() { + let mut ids = KBox::init(BufferIds::new(), GFP_KERNEL).unwrap(); + for _ in 0..8 { + let mut clients = KVec::with_capacity(BUFFER_IDS, GFP_KERNEL).unwrap(); + for _ in 0..BUFFER_IDS { + clients.push(ids.allocate_render().unwrap().0, GFP_KERNEL).unwrap(); + } + assert_eq!(ids.allocate_render(), Err(ENOSPC)); + for (index, id) in clients.into_iter().enumerate() { + if index % 2 == 0 { + // Failed constructor never installed the graph owner. + ids.release(id).unwrap(); + } else { + ids.reserve_owner(id).unwrap(); + ids.release(id).unwrap(); + ids.acquire_owner(id).unwrap(); + // Close cannot return a still-owned failed command. + ids.unreserve_owner(id).unwrap(); + assert!(!ids.entries[id as usize].free()); + ids.release(id).unwrap(); + } + } + assert!(ids.entries.iter().all(|entry| entry.free())); + assert_eq!(ids.returned_count, BUFFER_IDS); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index ed0f3c71a4c93..64c88da8eca14 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -6,7 +6,7 @@ use super::compact_alias; use crate::g17::{ - fw::render::RenderPass, + fw::render::{RegisterPrograms, RenderPass}, object::{CpuMap, KernelObject}, }; use crate::{ @@ -25,6 +25,12 @@ fn aligned(value: usize, alignment: usize) -> Result { .ok_or(EOVERFLOW) } +/// The 25G83 tiler parameter block: the TPC and seven sub-buffers the tiling registers name, at +/// the default parameter-buffer sizes. Sizes are +/// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. +const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; +const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, 0x600, 0x80]; + /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { @@ -33,9 +39,18 @@ pub(crate) struct Layout { pub(crate) used: usize, pub(crate) tpc_size: usize, pub(crate) tvb_blocks: usize, + /// Offsets of the tiler parameter sub-buffers in the TPC object (25G83), `None` for an + /// empty one; the TPC is entry 0. + gtp: [Option; 8], + /// Bytes of the TPC object the command uses. + tpc_end: usize, } impl Layout { - pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { + pub(crate) fn new( + pass: &RenderPass, + clusters: u32, + programs: RegisterPrograms, + ) -> Result { let width = pass.width as usize; let height = pass.height as usize; let layers = pass.layers as usize; @@ -90,16 +105,41 @@ impl Layout { ); } tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); + let mut gtp = [None; 8]; + let mut tpc_end = tpc_size; + if programs == RegisterPrograms::V25G83 { + let sizes = [ + tpc_size, + tilemap_size.checked_mul(64).ok_or(EOVERFLOW)?, + if layers > 1 { 0x4000 } else { 0 }, + 0, + GTP_FIXED_SIZES[0], + GTP_FIXED_SIZES[1], + GTP_FIXED_SIZES[2], + GTP_FIXED_SIZES[3], + ]; + let mut position = 0; + for ((offset, size), alignment) in gtp.iter_mut().zip(sizes).zip(GTP_ALIGNMENTS) { + position = aligned(position, alignment)?; + if size != 0 { + *offset = Some(position); + } + position = position.checked_add(size).ok_or(EOVERFLOW)?; + } + tpc_end = position; + } Ok(Self { layer_offset, heap_offset, used, tpc_size, tvb_blocks, + gtp, + tpc_end, }) } pub(crate) fn tpc_capacity(&self) -> Result { - Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) @@ -169,6 +209,21 @@ impl Target { self.tpc.address } + /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty + /// or absent one. The TPC object holds them after the TPC. + pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { + if layout.tpc_end > self.tpc.backing.size() { + return Err(ERANGE); + } + let mut addresses = [0; 8]; + for (address, offset) in addresses.iter_mut().zip(layout.gtp) { + if let Some(offset) = offset { + *address = self.tpc.address.checked_add(offset as u64).ok_or(EOVERFLOW)?; + } + } + Ok(addresses) + } + /// Only the span consumed by this render is cleared. A small render does /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs new file mode 100644 index 0000000000000..bcbbb2e7e875d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Name capacity refusals at their source, before a common submit or fence +//! failure handler loses their provenance. Counters are device-lifetime +//! independent diagnostics, never admission or ownership state. + +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::prelude::*; + +#[derive(Clone, Copy, Debug)] +pub(super) enum Site { + ComputeReserve, + PhysicalQids, + RenderSlots, + LogicalQueues, + ComputeSlots, + QosIds, + ComputeItems, + RetiredRenderBindings, + RenderPoolIds, + RenderWorkSlots, + ParameterBuffers, + MetricsIds, + LowerVa, + UpperVa, + AliasVa, + ExecutionContexts, +} + +const SITE_COUNT: usize = Site::ExecutionContexts as usize + 1; +static REFUSALS: [AtomicU64; SITE_COUNT] = [const { AtomicU64::new(0) }; SITE_COUNT]; + +/// First and power-of-two refusals per site keep a pressure storm bounded in +/// the log. This does not change whether the caller retries or rejects work. +#[track_caller] +pub(super) fn refused(site: Site) -> Error { + let count = REFUSALS[site as usize].fetch_add(1, Ordering::Relaxed).wrapping_add(1); + if count.is_power_of_two() { + let at = core::panic::Location::caller(); + pr_warn!("G17 capacity refusal {:?} count={} at {}:{}\n", site, count, + at.file().rsplit("asahi/").next().unwrap_or(at.file()), at.line()); + } + ENOSPC +} + +/// Read-only diagnostics hold the live device session while sampling its pools. +/// They never wake firmware, reclaim a resource, or change admission state. +pub(super) struct View(pub(super) kernel::sync::Arc); + +impl kernel::debugfs::Writer for View { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "version=1 sampled_ns={} crashed={}", super::now_ns(), + u8::from(self.0.crashed.load(Ordering::Acquire)))?; + let uat = { + let state = self.0.state.lock(); + let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; + writeln!(f, "online=1")?; + firmware.queues.write_capacity(f)?; + firmware.pool.write_capacity(f)?; + writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; + firmware.uat.clone() + }; + // Mapping destruction can issue firmware flushes while holding VM + // locks. Never take a VM/TTBAT lock with the device mutex held here. + // The retained UAT owns valid allocator metadata even during shutdown. + writeln!(f, "execution_contexts used={} total={}", uat.execution_contexts_used(), crate::mmu::MAX_EXECUTION_CONTEXTS)?; + let mut upper = uat.geometry().kernel_range(); + upper.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + for (name, vm, range) in [ + ("dynamic_lower_bytes", uat.kernel_lower_vm(), crate::hw::t8140::dynamic::LOWER), + ("dynamic_upper_bytes", uat.kernel_vm(), upper), + ] { + match vm.capacity(range.clone()) { + Ok((used, largest)) => writeln!(f, "{} used={} total={} largest_hole={}", name, used, range.end - range.start, largest)?, + Err(error) => writeln!(f, "{} unavailable={:?}", name, error)?, + } + } + for (site, count) in [ + Site::ComputeReserve, Site::PhysicalQids, Site::RenderSlots, Site::LogicalQueues, + Site::ComputeSlots, Site::QosIds, Site::ComputeItems, Site::RetiredRenderBindings, + Site::RenderPoolIds, Site::RenderWorkSlots, Site::ParameterBuffers, Site::MetricsIds, + Site::LowerVa, Site::UpperVa, Site::AliasVa, Site::ExecutionContexts, + ].iter().zip(REFUSALS.iter()) { + writeln!(f, "refusals site={:?} count={}", site, count.load(Ordering::Relaxed))?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 3dfa760969224..c65f7de6ac915 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -12,6 +12,8 @@ use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; use kernel::{prelude::*, uapi}; +const _: () = assert!(uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_AS_BUILD == 1 << 2); + const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; @@ -121,7 +123,7 @@ impl<'a> Parser<'a> { let bytes = self.bytes.get(start..end).ok_or(EINVAL)?; self.bytes = &self.bytes[end..]; let barriers = [header.vdm_barrier, header.cdm_barrier]; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER | uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { if barriers.iter().zip(self.counts).any(|(&barrier, count)| { @@ -131,7 +133,7 @@ impl<'a> Parser<'a> { { return Err(EINVAL); } - let payload = if header.cmd_type as u32 + let payload = if u32::from(header.cmd_type) == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { self.counts[0] += 1; @@ -238,7 +240,7 @@ impl UscWindow { } require_range( space, - self.base + (tagged_offset & !7) as u64, + self.base + u64::from(tagged_offset & !7), 4, Access::Read, ) @@ -306,12 +308,14 @@ fn surface( #[allow(clippy::large_enum_variant)] pub(crate) enum Validated { Render { + long_job: bool, pass: RenderPass, ray_query: [RayQueryStage; 2], attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + long_job: bool, intersection: IntersectionTables, scratch: ScratchRequest, usc_base: u64, @@ -323,6 +327,19 @@ pub(crate) enum Validated { }, } +impl Validated { + pub(crate) fn long_job(&self) -> bool { + match self { + Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job, + } + } + pub(crate) fn allow_long_job(&mut self) { + match self { + Self::Render { long_job, .. } | Self::Compute { long_job, .. } => *long_job = true, + } + } +} + impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { @@ -334,6 +351,7 @@ impl Payload { pub(crate) fn validate( &self, window: &UscWindow, + variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { match self { @@ -407,7 +425,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, - few_primitives: cmd.flags & FEW_PRIMITIVES != 0, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, @@ -440,6 +458,11 @@ impl Payload { rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; for stage in ray_query { + if (stage.intersection.enabled() || stage.scratch.enabled()) + && !variant.ray_traversal + { + return Err(refuse_ray_traversal()); + } if stage.intersection.enabled() { require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; @@ -447,6 +470,8 @@ impl Payload { } } Ok(Validated::Render { + long_job: super::timeout::select(false, + ray_query.map(|stage| stage.intersection.enabled()), false), pass, ray_query: *ray_query, attachments: *attachments, @@ -454,8 +479,8 @@ impl Payload { }) } Self::Compute(cmd, attachments, intersection) => { - if cmd.flags != 0 - || cmd.cdm_ctrl_stream_base & 3 != 0 + let as_build = super::timeout::as_build(cmd.flags).ok_or(EINVAL)?; + if cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base || !helper_is_empty(&cmd.helper) @@ -470,14 +495,19 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; + let scratch = ScratchRequest::new(cmd.scs_layout)?; + if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { + return Err(refuse_ray_traversal()); + } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; if code != 0 { require_range(space, code, 64, Access::Read)?; } } Ok(Validated::Compute { + long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), intersection: *intersection, - scratch: ScratchRequest::new(cmd.scs_layout)?, + scratch, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, @@ -489,3 +519,9 @@ impl Payload { } } } + +/// Ray-traversal work on a firmware build that does not run it ([`super::fw::Variant`]). +fn refuse_ray_traversal() -> Error { + kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); + Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) +} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index a3888f98231db..4f5b87bc853dc 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -28,7 +28,7 @@ pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { // Keep the existing GPU alias alive along with its prepared CPU view. - _mapping: Arc, + _mapping: Arc, cpu: mmu::TimestampMapping, } @@ -144,7 +144,8 @@ impl HostOutput { } } -/// Firmware writes two aligned words in a fresh write-combined page per compute command. +/// Firmware writes two aligned words in a write-combined page of its own per +/// compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. mapping: mmu::KernelMapping, @@ -166,6 +167,84 @@ impl ComputeTimestamps { } } +/// Compute timestamp pages of one context, kept mapped for its next commands. +/// +/// A fresh page costs a GEM object, its write-combined kernel mapping and a +/// GPU mapping on the submission path, and their teardown when the command's +/// completion is dropped. A page comes back here only from that drop, which +/// follows exact retirement or a kill witness, so the firmware no longer +/// writes it; both words are cleared before the page is handed out again. +#[pin_data] +pub(crate) struct ComputeTimestampCache { + #[pin] + pages: Mutex>, +} + +impl ComputeTimestampCache { + /// Pages retained per context; deeper pipelines allocate the rest. + const RETAINED: usize = 8; + + pub(crate) fn new() -> Result> { + let pages = KVec::with_capacity(Self::RETAINED, GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { + pages <- new_mutex!(pages, "G17 compute timestamp pages"), + }), + GFP_KERNEL, + ) + } + + fn take(self: &Arc, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let reused = self.pages.lock().pop(); + let page = match reused { + Some(page) => { + page.object.dword(0)?.store(0, Ordering::Relaxed); + page.object.dword(8)?.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + page + } + None => ComputeTimestamps::new(dev, vm)?, + }; + Ok(PooledTimestamps { + page: Some(page), + cache: self.clone(), + }) + } + + fn put(&self, page: ComputeTimestamps) { + let rejected = { + let mut pages = self.pages.lock(); + pages.push_within_capacity(page).err() + }; + // Teardown of a page beyond the retained count runs unlocked. + drop(rejected); + } +} + +/// A compute timestamp page lent to one command, returned to its context's +/// cache when the command's completion is dropped. +struct PooledTimestamps { + page: Option, + cache: Arc, +} + +impl PooledTimestamps { + fn page(&self) -> Result<&ComputeTimestamps> { + self.page.as_ref().ok_or(EINVAL) + } +} + +impl Drop for PooledTimestamps { + fn drop(&mut self) { + if let Some(page) = self.page.take() { + self.cache.put(page); + } + } +} + +/// Republications of one command whose earlier publications firmware dropped unexecuted. +pub(crate) const MAX_REPLAYS: u8 = 3; + #[pin_data] pub(crate) struct Completion { status: Arc, @@ -173,7 +252,7 @@ pub(crate) struct Completion { // Present only when this command's public result may wait for a render // predecessor's host writes. DRM credits must follow the terminal result. terminal: Option>, - compute: Option, + compute: Option, feed: Option>, #[pin] vm_job: Mutex>, @@ -185,11 +264,18 @@ pub(crate) struct Completion { timeout_error: AtomicI32, replays: AtomicU8, replay_timeout: AtomicBool, + /// A recovery blamed this command. Unlike `class`, which a later + /// classification can change, this never clears. + blamed: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } impl Completion { + #[expect( + clippy::too_many_arguments, + reason = "Completion construction takes distinct queue, context, and lifetime owners." + )] pub(crate) fn new( dev: &AsahiDrmDevice, contexts: &FenceContexts, @@ -216,7 +302,7 @@ impl Completion { HostOutput::Immediate(output) }; let compute = if compute { - Some(ComputeTimestamps::new(dev, context.vm())?) + Some(context.compute_timestamps().take(dev, context.vm())?) } else { None }; @@ -235,6 +321,7 @@ impl Completion { timeout_error: AtomicI32::new(0), replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), + blamed: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -278,14 +365,14 @@ impl Completion { } pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + Ok(self.compute.as_ref().ok_or(EINVAL)?.page()?.mapping.iova()) } /// The caller has proved that firmware retired this command, or has /// received its scheduler kill acknowledgement with recovery closed. /// These are the command's public timestamp words, not queue-private state. pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let words = [ object.dword(0)?.load(Ordering::Relaxed), object.dword(8)?.load(Ordering::Relaxed), @@ -305,7 +392,7 @@ impl Completion { let accepted = self .replays .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { - (count < 3).then_some(count + 1) + (count < MAX_REPLAYS).then_some(count + 1) }) .is_ok(); if accepted { @@ -335,7 +422,7 @@ impl Completion { if self.signalled.load(Ordering::Acquire) { return Err(ECANCELED); } - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let object = &self.compute.as_ref().ok_or(EINVAL)?.page()?.object; let start = object.dword(0)?; let end = object.dword(8)?; start.store(0, Ordering::Relaxed); @@ -384,6 +471,20 @@ impl Completion { self.class.load(Ordering::Acquire) == 1 } + pub(crate) fn guilty(&self) -> bool { + self.class.load(Ordering::Acquire) == 2 + } + + /// A recovery blamed this command, whatever class it keeps: it may have caused the fault, + /// so it is never published again. + pub(crate) fn note_blamed(&self) { + self.blamed.store(true, Ordering::Release); + } + + pub(crate) fn blamed(&self) -> bool { + self.blamed.load(Ordering::Acquire) + } + /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -401,6 +502,12 @@ impl Completion { Ok(()) } + /// An ioctl publication that met contention returns its lease before the + /// scheduler publishes the command. Nothing referred to it yet. + pub(crate) fn detach_render_lease(&self) -> Option { + self.render.lock().take() + } + pub(crate) fn render_slot(&self) -> Option { self.render .lock() @@ -411,7 +518,7 @@ impl Completion { /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { - lease.quarantine(); + lease.quarantine(self.spared()); } } @@ -442,8 +549,11 @@ impl Completion { feed.note_render_pass(ticks[2], ticks[3]); } if let Err(error) = result { + // Spared work cancelled after another VM's recovery is not this client's device loss. if self.spared() { - self.status.report_failure(error); + if error != ECANCELED { + self.status.report_failure(error); + } } else if error != ECANCELED { self.status.record(error); } diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 6bc62a7d44bee..c93ab25d55ce3 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(ENOSPC); + return Err(super::capacity::refused(super::capacity::Site::QosIds)); } *used |= 1u128 << id; Ok(QosLease { @@ -121,6 +121,8 @@ pub(crate) struct Context { work_base: u64, render: Arc>>>, render_blocks: AtomicUsize, + /// Timestamp pages of retired compute commands, mapped in this context's VM. + compute_timestamps: Arc, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -194,12 +196,16 @@ impl Context { work_base, render, render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), + compute_timestamps: super::completion::ComputeTimestampCache::new()?, policy, }) } pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_timestamps(&self) -> &Arc { + &self.compute_timestamps + } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -387,7 +393,8 @@ impl WorkStateLease { } pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or(ENOSPC) + self.context.work.lock().render.get(pair).ok_or_else(|| + super::capacity::refused(super::capacity::Site::RenderWorkSlots)) } pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { self.context @@ -406,3 +413,37 @@ impl Drop for WorkStateLease { } } } + +impl QosIds { + pub(crate) fn used(&self) -> u32 { + (*self.used.lock() & !3u128).count_ones() + } +} + +#[kunit_tests(asahi_g17_qos_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn failed_client_construction_returns_every_qos_id() { + let ids = QosIds::new().unwrap(); + for _ in 0..8 { + let mut clients = KVec::with_capacity(126, GFP_KERNEL).unwrap(); + for _ in 0..126 { + clients.push(ids.acquire().unwrap(), GFP_KERNEL).unwrap(); + } + assert_eq!(ids.used(), 126); + assert!(matches!(ids.acquire(), Err(ENOSPC))); + drop(clients); + assert_eq!(ids.used(), 0); + assert_eq!(*ids.used.lock(), 3); + } + } +} + +impl Context { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>, scope: &str, index: usize) -> kernel::fmt::Result { + writeln!(f, "context scope={} index={} id={} work_used={} work_total={} render_used={} render_total=2", + scope, index, self.id(), self.states.used(), STATE_COUNT, self.work.lock().render.used()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 6c48367e2d686..595c7154d42ed 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -69,3 +69,12 @@ impl RenderSlots { Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) } } + +impl WorkStates { + pub(super) fn used(&self) -> usize { + self.live.iter().filter(|word| word.load(Ordering::Acquire) != FREE).count() + } +} +impl RenderSlots { + pub(super) fn used(&self) -> usize { self.pairs.iter().flatten().count() } +} diff --git a/drivers/gpu/drm/asahi/g17/data_segment.rs b/drivers/gpu/drm/asahi/g17/data_segment.rs new file mode 100644 index 0000000000000..02bcc66de8f51 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/data_segment.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2026 The Gravity Linux Contributors +// +// Derived from the crash buffer mapping of Gravity Linux's M4 GPU driver (CrashBuffer and +// BootOps::shmem_map in drivers/gpu/drm/asahi/g16.rs), written by Cody Ho: +// https://github.com/GravityLinux/linux/commit/bd25d194b0a295a75692a0e6caed3793603b9e65 +// Gravity's g16.rs is adapted from Niklas Sheth's linux-m4-integration prototype. + +//! Crash log buffers in the GPU firmware's data segment. +//! +//! A firmware that places its own crash log ([`super::CrashBuffers::FirmwareData`]) asks for it +//! by physical address inside its data segment, the `fw-data` reserved region, which the boot +//! loader keeps for the loaded firmware until reboot. + +use kernel::{ + c_str, device, + io::{ + self, + IoSysMap, + Region, // + }, + prelude::*, +}; + +/// `size` bytes at `offset` of the firmware's data segment, mapped write-back. +pub(super) struct DataSegmentLog { + mem: io::mem::Mem, + offset: usize, + size: usize, +} + +impl DataSegmentLog { + /// Maps the `size` bytes at physical address `iova` that the firmware asked for, which + /// must lie inside the `fw-data` region of `dev`. + pub(super) fn map(dev: &device::Device, iova: usize, size: usize) -> Result { + let node = dev.of_node().ok_or(EINVAL)?; + let res = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; + let offset = iova.checked_sub(res.start().try_into()?).ok_or(EINVAL)?; + if offset.checked_add(size).ok_or(EINVAL)? > res.size().try_into()? { + return Err(EINVAL); + } + // SAFETY: The boot loader reserved this RAM for the loaded GPU firmware, which keeps + // it until reboot; only the requested subrange is exposed. + let mem = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; + Ok(Self { mem, offset, size }) + } + + /// The CPU view of the buffer. + pub(super) fn buf(&mut self) -> Result> { + // SAFETY: map() checked that the range lies inside the mapping. + let base = unsafe { self.mem.ptr().add(self.offset) }; + let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, self.size)?; + // SAFETY: The range is mapped RAM for as long as this borrow of the mapping. + Ok(IoSysMap::Sys(unsafe { io::SysMem::new(ptr) })) + } +} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 5abd494eeb1f5..0bc1d5024566e 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -17,6 +17,14 @@ pub(crate) fn inherited_error(error: Error) -> Error { } } +/// A replay's original host inputs must still be signalled. Treat an unmet +/// input as a failed replay, never as a wait under the device mutex. +pub(crate) fn replay_status(status: i32) -> Result { + if status == 0 { Err(EBUSY) } + else if status < 0 { Err(inherited_error(Error::from_errno(status))) } + else { Ok(()) } +} + pub(crate) struct Frontier { dependencies: [KickDependency; KICK_BARRIERS_MAX], sequences: [u64; KICK_BARRIERS_MAX], diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index 9e4e0253b7c81..9508a827be4b1 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -11,6 +11,8 @@ use kernel::prelude::*; /// The bounded shared-ring transport; recognized malformed entries return errors after consume. pub(super) trait Transport { + /// The firmware instances of this GPU: the primary, and the secondary where one exists. + fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); fn drain( @@ -81,10 +83,10 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.trace(role); } - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.statistics(role); } let mut masks = [0; 2]; @@ -110,7 +112,7 @@ pub(super) fn service(host: &mut H) -> Result { return Err(EIO); } } - if !host.control_backpressured() { + if !host.control_backpressured() && host.roles().contains(&Role::Secondary) { batch(host, Role::Secondary, &mut masks)?; } host.retain_render_completions(masks); diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 4cd05fde16581..0045dff413922 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -147,6 +147,10 @@ impl Requests { } } + /// Add a policy to the pinned requests. + /// + /// # Safety + /// /// The caller holds the policy's read or write lock and keeps it alive. /// Policy removal must subsequently pass through this device's notifier. unsafe fn add(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { @@ -179,7 +183,7 @@ impl Requests { this.maxima[index] = (*policy).cpuinfo.max_freq; } this.policies[index] = policy; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } } @@ -193,7 +197,7 @@ impl Requests { let value = if enabled { (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 } else { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 + bindings::FREQ_QOS_MIN_DEFAULT_VALUE }; if this.applied[index] != value { // SAFETY: This slot is registered, pinned and exclusively locked; @@ -211,7 +215,7 @@ impl Requests { this.applied[index] = value; } } - applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } applied } @@ -235,7 +239,7 @@ impl Requests { } this.policies[index] = ptr::null_mut(); this.maxima[index] = 0; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE; } fn clear(mut self: Pin<&mut Self>) { @@ -364,6 +368,13 @@ impl Feed { } } + /// Process a cpufreq policy notification. + /// + /// # Safety + /// + /// `block` must be this pinned Feed's registered notifier, and `data` must + /// point to the live policy supplied by the cpufreq notifier core. + /// CREATE must hold its write lock; REMOVE must precede policy release. #[cfg(CONFIG_CPU_FREQ)] unsafe extern "C" fn policy_event( block: *mut bindings::notifier_block, @@ -387,12 +398,12 @@ impl Feed { // held and guarantees REMOVE before the core releases its storage. unsafe { requests.as_mut().add(policy) }; } else { - return bindings::NOTIFY_DONE as _; + return bindings::NOTIFY_DONE as i32; } let enabled = feed.detector.wanted(Self::now()); let applied = requests.as_mut().apply(enabled); feed.applied.store(applied, Ordering::Relaxed); - bindings::NOTIFY_OK as _ + bindings::NOTIFY_OK as i32 } fn now() -> u64 { diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c9c9ebda7f6bc..3b31c3cead064 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -59,7 +59,7 @@ impl RenderId { } } } - Err(ENOSPC) + Err(super::capacity::refused(super::capacity::Site::RenderPoolIds)) } } impl Drop for RenderId { @@ -69,19 +69,84 @@ impl Drop for RenderId { } } +/// Free-list backing is by far the largest user of the shared lower GPU VA +/// window, and placing it populates every page before the window is searched. +/// After a refused placement, builders waiting for capacity would allocate and +/// free tens of MiB per attempt while nothing in the window changed. A refused +/// size is therefore refused again without allocating until free-list backing +/// is freed, except for one probe per interval across all callers. +static LOWER_FREED: AtomicU64 = AtomicU64::new(0); +/// `LOWER_FREED + 1` at the latest refusal, or zero after a placement fits. +static LOWER_REFUSED: AtomicU64 = AtomicU64::new(0); +static LOWER_REFUSED_SIZE: AtomicU64 = AtomicU64::new(0); +static LOWER_PROBE_AT: AtomicU64 = AtomicU64::new(0); +const LOWER_PROBE_NS: u64 = 100_000_000; + +/// The window refused free-list backing and nothing was freed since. +pub(crate) fn lower_pressure() -> bool { + let refused = LOWER_REFUSED.load(Ordering::Acquire); + refused != 0 && refused == LOWER_FREED.load(Ordering::Acquire).wrapping_add(1) +} + +/// Place free-list backing of `size` bytes unless the same or a smaller size +/// was refused with nothing freed since; `place` allocates and maps it. +fn place_lower(size: usize, place: impl FnOnce() -> Result) -> Result { + let size = size as u64; + if lower_pressure() && size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + let now = super::now_ns(); + let at = LOWER_PROBE_AT.load(Ordering::Relaxed); + if now < at + || LOWER_PROBE_AT + .compare_exchange(at, now + LOWER_PROBE_NS, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return Err(ENOSPC); + } + } + match place() { + Err(error) if error == ENOSPC => { + LOWER_REFUSED_SIZE.store(size, Ordering::Relaxed); + LOWER_PROBE_AT.store(super::now_ns() + LOWER_PROBE_NS, Ordering::Relaxed); + LOWER_REFUSED.store( + LOWER_FREED.load(Ordering::Acquire).wrapping_add(1), + Ordering::Release, + ); + Err(error) + } + Ok(placed) => { + if size >= LOWER_REFUSED_SIZE.load(Ordering::Relaxed) { + LOWER_REFUSED.store(0, Ordering::Release); + } + Ok(placed) + } + Err(error) => Err(error), + } +} + pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, } +impl Drop for Pages { + fn drop(&mut self) { + LOWER_FREED.fetch_add(1, Ordering::AcqRel); + } +} impl Pages { fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let mut backing = alloc.lower( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - )?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let backing = place_lower(size, || { + alloc.lower( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + ) + })?; + Self::from_backing(backing, blocks, layout) + } + fn from_backing(mut backing: KernelObject, blocks: usize, layout: RunLayout) -> Result { let base = backing.gpu_va(); backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; Ok(Self { @@ -100,7 +165,7 @@ impl Pages { pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } - fn page_list_va(&self) -> u64 { + pub(crate) fn page_list_va(&self) -> u64 { self.backing.gpu_va() } fn run_list_va(&self) -> u64 { @@ -144,14 +209,17 @@ impl RenderBacking { if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { return Err(EINVAL); } - let pages = Pages::new(alloc, blocks, RunLayout::Render)?; - let address = pages.page_list_va(); - let global = pages - .backing - .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; - let client = pages - .backing - .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + let size = TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE; + let (backing, [global, client]) = place_lower(size, || { + alloc.lower_with_aliases( + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + [render_global, client], + ) + })?; + let pages = Pages::from_backing(backing, blocks, RunLayout::Render)?; Ok(Self { _client: client, _global: global, @@ -522,6 +590,18 @@ impl ComputePool { if self.pages.is_some() { return Err(EBUSY); } + self.write_pages(&pages)?; + self.pages = Some(pages); + Ok(()) + } + /// Preserve both sets of pages on failure. The caller owns the consumed + /// release and returns old pages for destruction outside the device mutex. + pub(crate) fn replace_pages(&mut self, pages: &mut Option) -> Result { + self.write_pages(pages.as_ref().ok_or(EINVAL)?)?; + core::mem::swap(&mut self.pages, pages); + Ok(()) + } + fn write_pages(&mut self, pages: &Pages) -> Result { let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; let mut bytes = [0; 8]; for (index, byte) in bytes.iter_mut().enumerate() { @@ -540,9 +620,9 @@ impl ComputePool { header.unk_54[..8].copy_from_slice(&bytes); self.control.write(0, header)?; fence(Ordering::SeqCst); - self.pages = Some(pages); Ok(()) } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { let pages = self.pages.as_mut().ok_or(EAGAIN)?; let address = pages.page_list_va(); @@ -742,3 +822,30 @@ pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Res let ahead = cursor.wrapping_sub(consumer) & (slots - 1); Ok(ahead == 0 || ahead > pending) } + +impl RenderIds { + pub(crate) fn used(&self) -> u32 { + self.used[0].load(Ordering::Acquire).count_ones() + + (self.used[1].load(Ordering::Acquire) & !(1u64 << 63)).count_ones() + } +} + +#[kunit_tests(asahi_g17_render_id_capacity)] +mod capacity_tests { + use super::*; + + #[test] + fn rejected_pool_construction_returns_all_render_ids() { + let ids = Arc::new(RenderIds::new(), GFP_KERNEL).unwrap(); + for _ in 0..8 { + let mut pools = KVec::with_capacity(127, GFP_KERNEL).unwrap(); + for _ in 0..127 { + pools.push(RenderId::new(&ids).unwrap(), GFP_KERNEL).unwrap(); + } + assert_eq!(ids.used(), 127); + assert!(matches!(RenderId::new(&ids), Err(ENOSPC))); + drop(pools); + assert_eq!(ids.used(), 0); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs index ec6797637bdbe..9d430d50db365 100644 --- a/drivers/gpu/drm/asahi/g17/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -161,7 +161,7 @@ impl Graph { } let scratch = args.scratch.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; let discard = args.discard.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; - if scratch > u32::MAX as u64 || discard > u32::MAX as u64 { + if scratch > u64::from(u32::MAX) || discard > u64::from(u32::MAX) { return Err(EINVAL); } let stats = offset_va(args.fw_va, (CONTROL + 0x40) as u64)?; @@ -235,7 +235,7 @@ pub(crate) fn scene_registers(index: usize, scratch: u64, metrics: u64) -> Resul .ok_or(EINVAL)? .checked_add((index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE) .ok_or(EOVERFLOW)?; - if scratch > u32::MAX as u64 || scratch & 0xf != 0 { + if scratch > u64::from(u32::MAX) || scratch & 0xf != 0 { return Err(EINVAL); } let metric = offset_va(metrics, (index * 4) as u64)?; diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index c4cf6960db296..524132c4ae7be 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -498,7 +498,7 @@ impl WorkSlot { data_master: data_master as u32, items_end, qid, - first: first as u8, + first: u8::from(first), }) } } diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 0633da7f43541..0cb7304e30677 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -67,10 +67,10 @@ impl ScratchRequest { /// guard; the per-register words are the native formulas, so they are /// checked here against Apple's own values before they reach hardware. pub(crate) fn new(layout: u64) -> Result { - let depth = layout & u16::MAX as u64; - let cliques = (layout >> 24) & u8::MAX as u64; - let user = (layout >> 32) & u16::MAX as u64; - let driver = (layout >> 48) & u16::MAX as u64; + let depth = layout & u64::from(u16::MAX); + let cliques = (layout >> 24) & u64::from(u8::MAX); + let user = (layout >> 32) & u64::from(u16::MAX); + let driver = (layout >> 48) & u64::from(u16::MAX); if user == 0 && driver == 0 { return Ok(Self { pages: 0, @@ -152,13 +152,42 @@ const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; /// Offsets in the preempt buffer of the four state words the program names. const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; -/// Offsets in the operand-state buffer of the regions the program names. -const OPERAND_STATE_10229: u64 = 0x12800; -const OPERAND_STATE_140A8: u64 = 0x13000; -const OPERAND_STATE_10099: u64 = 0x9405; -const OPERAND_STATE_10091: u64 = 0x12400; -const OPERAND_STATE_0A5C1: u64 = 0x0005; -const OPERAND_STATE_0A5C9: u64 = 0x9000; +/// Layout of a queue's operand-state buffer, which depends on the GPU's core layout: offsets of +/// the regions the program names (two preemption banks of three 0x800-byte records per core, +/// tagged | 5 at their start, then their end, then the per-cluster tails) and its size. +pub(crate) struct OperandState { + r10229: u64, + r140a8: u64, + r10099: u64, + r10091: u64, + r0a5c1: u64, + r0a5c9: u64, + /// Size of the buffer. + pub(crate) size: usize, +} + +impl OperandState { + /// One cluster of six cores (A18 Pro). + pub(crate) const ONE_BY_SIX: OperandState = OperandState { + r10229: 0x12800, + r140a8: 0x13000, + r10099: 0x9405, + r10091: 0x12400, + r0a5c1: 0x0005, + r0a5c9: 0x9000, + size: 0x14000, + }; + /// Two clusters of ten cores (M4 Pro): bank stride 0x1e400, tails 0x1000. + pub(crate) const TWO_BY_TEN: OperandState = OperandState { + r10229: 0x3c800, + r140a8: 0x3d800, + r10099: 0x1e405, + r10091: 0x3c400, + r0a5c1: 0x0005, + r0a5c9: 0x1e000, + size: 0x44000, + }; +} /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full @@ -283,6 +312,8 @@ pub(crate) struct ComputeArgs { pub(crate) preempt_va: u64, /// GPU address of the operand-state buffer, as seen by the work's VM. pub(crate) operand_state_va: u64, + /// Layout of the operand-state buffer. + pub(crate) operand: &'static OperandState, /// GPU address of the private-memory usage record, as seen by the work's /// VM. pub(crate) usage_va: u64, @@ -305,6 +336,7 @@ pub(crate) struct ComputeArgs { fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; + let state = args.operand; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); let intersection = args.intersection.words(); @@ -347,14 +379,14 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x0a351, ids), RegisterWrite::new(0x10201, key), RegisterWrite::new(0x10428, key), - RegisterWrite::new(0x14028, args.free_list_slot as u64), + RegisterWrite::new(0x14028, u64::from(args.free_list_slot)), RegisterWrite::new(0x14070, args.usage_va | 1), - RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), - RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x10229, offset_va(operand, state.r10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, state.r140a8)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r0a5c9)?), RegisterWrite::new(0x1a440, UNK_1A440_FINAL), RegisterWrite::new(0x0a599, UNK_0A599), ]) @@ -364,11 +396,12 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS /// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { let operand = args.operand_state_va; + let state = args.operand; Ok([ - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r0a5c9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r10091)?), ]) } diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 1f25bd0fc1c0d..3092e599caa84 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -27,6 +27,8 @@ use kernel::prelude::*; use super::clear; +pub(crate) mod v25g83; + /// Interface version at root `+0x00`; the firmware rejects any other. pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; @@ -219,6 +221,21 @@ impl SecondaryRoot { config_va, } } + + /// Returns the root of a single instance that carries both roles: the primary root, with + /// status B, extended by the power-state and configuration pointers. + pub(crate) fn merged( + args: &RootArgs, + status_b_va: u64, + power_state_va: u64, + config_va: u64, + ) -> Self { + Self { + root: Root::new(Role::Primary, args, status_b_va), + power_state_va, + config_va, + } + } } /// Offset of the secondary root from the primary root. Both live in one @@ -591,31 +608,42 @@ pub(crate) struct RegisterWindow { pub(crate) phys: u64, /// GPU address of the window in the firmware MMIO aperture. pub(crate) va: u64, - /// Size in bytes; need not be a multiple of the page size. + /// Size in bytes of one instance; need not be a multiple of the page size. pub(crate) size: u32, /// Value of [`RegisterMapEntry::unk_18`]. pub(crate) unk_18: u64, /// Value of [`RegisterMapEntry::unk_20`]. pub(crate) unk_20: u32, + /// Number of instances of the block. The firmware finds instance `n` at + /// `va + n * size`; it lives at `phys + n * phys_stride`. + pub(crate) instances: u32, + /// Distance between the physical addresses of consecutive instances. + pub(crate) phys_stride: u64, } impl RegisterWindow { - /// Returns the page-aligned physical address, GPU address and length the - /// window must be mapped with, or `None` if its physical and GPU - /// addresses differ within a page or the mapped span overflows. - pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { + /// Returns the page-aligned physical address, GPU address and length that + /// instance `instance` of the window must be mapped with, or `None` if its + /// physical and GPU addresses differ within a page or the mapped span + /// overflows. + pub(crate) fn mapping(&self, instance: u32) -> Option<(u64, u64, u64)> { let mask = PAGE_SIZE as u64 - 1; - if self.phys & mask != self.va & mask { + let phys = self + .phys + .checked_add(self.phys_stride.checked_mul(u64::from(instance))?)?; + let va = self + .va + .checked_add(u64::from(self.size).checked_mul(u64::from(instance))?)?; + if phys & mask != va & mask { return None; } - let start = self.va & !mask; - let span = self - .va - .checked_add(self.size as u64)? + let start = va & !mask; + let span = va + .checked_add(u64::from(self.size))? .checked_sub(start)? .checked_add(mask)? & !mask; - Some((self.phys & !mask, start, span)) + Some((phys & !mask, start, span)) } } @@ -629,9 +657,9 @@ pub(crate) struct RegisterMapEntry { pub(crate) phys: u64, /// GPU address of the window. pub(crate) va: u64, - /// Window size in bytes. + /// Size in bytes of all instances of the window. pub(crate) total_size: u32, - /// Window size in bytes, again. + /// Size in bytes of one instance. pub(crate) element_size: u32, /// Per-window value, see [`RegisterWindow::unk_18`]. pub(crate) unk_18: u64, @@ -650,7 +678,7 @@ impl RegisterMapEntry { Self { phys: window.phys, va: window.va, - total_size: window.size, + total_size: window.size * window.instances, element_size: window.size, unk_18: window.unk_18, unk_20: window.unk_20, @@ -676,14 +704,19 @@ pub(crate) const PERF_STATES: usize = 11; /// Voltage lanes of a performance table. const PERF_LANES: usize = 16; +/// Most voltage cells of an operating point: one per GPU cluster, at most one per lane. +pub(crate) const OPP_CELLS: usize = PERF_LANES; + /// One GPU operating point, from the device tree, in performance-state /// order. #[derive(Copy, Clone, Debug)] pub(crate) struct Opp { /// Frequency in Hz. pub(crate) freq_hz: u64, - /// GPU core voltage in µV. - pub(crate) volt_uv: u32, + /// GPU core voltage in µV: one cell for the whole GPU, or one per cluster. + pub(crate) volt_uv: [u32; OPP_CELLS], + /// Number of valid cells in [`Self::volt_uv`]. + pub(crate) cells: usize, } /// Per-SoC parts of the performance tables that the operating points do not @@ -728,20 +761,23 @@ static_assert!(size_of::() == 0x704); #[derive(Copy, Clone)] struct PerfState { freq_mhz: u32, - core_mv: u32, - sram_mv: u32, + core_mv: [u32; OPP_CELLS], + sram_mv: [u32; OPP_CELLS], + cells: usize, } impl PerfGroup { - /// Fills the table. `lanes` is the number of voltage lanes holding each - /// state's voltages; the remaining lanes stay zero. + /// Fills the table. With one voltage cell, `lanes` is the number of voltage lanes + /// holding each state's voltages; with one cell per cluster, lane `n` holds cluster + /// `n`'s. The remaining lanes stay zero. fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { self.unk_0 = 10; for (index, state) in states.iter().enumerate() { self.freq_mhz[index] = state.freq_mhz; - for lane in 0..lanes { - self.core_mv[index][lane] = state.core_mv; - self.sram_mv[index][lane] = state.sram_mv; + for lane in 0..if state.cells == 1 { lanes } else { state.cells } { + let cell = if state.cells == 1 { 0 } else { lane }; + self.core_mv[index][lane] = state.core_mv[cell]; + self.sram_mv[index][lane] = state.sram_mv[cell]; } } } @@ -755,16 +791,18 @@ fn perf_states( ) -> Result<[PerfState; PERF_STATES]> { let mut states = [PerfState { freq_mhz: 0, - core_mv: 0, - sram_mv: 0, + core_mv: [0; OPP_CELLS], + sram_mv: [0; OPP_CELLS], + cells: 0, }; PERF_STATES]; for (state, &index) in states.iter_mut().zip(map) { let opp = opps.get(index as usize).ok_or(EINVAL)?; - let core_mv = opp.volt_uv / 1000; + let core_mv = opp.volt_uv.map(|uv| uv / 1000); *state = PerfState { freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, core_mv, - sram_mv: core_mv.max(sram_min_mv), + sram_mv: core_mv.map(|mv| mv.max(sram_min_mv)), + cells: opp.cells, }; } Ok(states) @@ -1169,6 +1207,35 @@ pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; /// scheduling pass. pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; +/// The values of region C that differ between SoCs and firmware builds. +pub(crate) struct RegionCConfig { + /// A18 Pro: 0xffff. + pub(crate) unk_54: u16, + /// A18 Pro: 1 in every entry. + pub(crate) unk_78: [u32; 3], + /// Highest GPU power, in mW (A18 Pro: 9550). + pub(crate) max_power_mw: u32, + /// A18 Pro: 0x3000_001d. + pub(crate) unk_9c: u32, + /// A18 Pro: 0x3100_0002. + pub(crate) unk_a0: u32, + /// A18 Pro: 0x7d0 (2000). + pub(crate) unk_7c4: u32, + /// A18 Pro: zero. + pub(crate) unk_9e0: [u32; 8], +} + +/// Region C of the A18 Pro firmware (26A428). +pub(crate) const REGION_C_26A428: RegionCConfig = RegionCConfig { + unk_54: 0xffff, + unk_78: [1; 3], + max_power_mw: 9550, + unk_9c: 0x3000_001d, + unk_a0: 0x3100_0002, + unk_7c4: 0x7d0, + unk_9e0: [0; 8], +}; + /// Region C: firmware policy settings shared by both instances. /// /// Three fields are live. The host updates [`Self::effort`] while it submits @@ -1197,7 +1264,7 @@ pub(crate) struct RegionC { /// Required: 0x78 (120). pub(crate) unk_3c: u32, pub(crate) unk_40: [u8; 0x14], - /// Required: 0xffff. + /// See [`RegionCConfig::unk_54`]. pub(crate) unk_54: u16, /// Required: 0x28 (40). pub(crate) unk_56: u16, @@ -1213,10 +1280,10 @@ pub(crate) struct RegionC { /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). pub(crate) context_switch_timer_multiplier: u8, pub(crate) unk_6b: [u8; 0xd], - /// Required: 1 in every entry. + /// See [`RegionCConfig::unk_78`]. pub(crate) unk_78: [u32; 3], - /// Required: 0x254e (9550). - pub(crate) unk_84: u32, + /// Highest GPU power, in mW. + pub(crate) max_power_mw: u32, /// Required: 0x3e8 (1000). pub(crate) unk_88: u32, /// Required: 0x3e8 (1000). @@ -1224,9 +1291,9 @@ pub(crate) struct RegionC { pub(crate) unk_90: [u8; 8], /// Required: 0x3e8 (1000). pub(crate) unk_98: u32, - /// Required: 0x3000_001d. + /// See [`RegionCConfig::unk_9c`]. pub(crate) unk_9c: u32, - /// Required: 0x3100_0002. + /// See [`RegionCConfig::unk_a0`]. pub(crate) unk_a0: u32, pub(crate) unk_a4: [u8; 0x40], /// Performance-floor request (live, see the type documentation). @@ -1234,7 +1301,7 @@ pub(crate) struct RegionC { /// Live; the host stores zero here whenever it stores [`Self::effort`]. pub(crate) unk_e8: u32, pub(crate) unk_ec: [u8; 0x6d8], - /// Required: 0x7d0 (2000). + /// See [`RegionCConfig::unk_7c4`]. pub(crate) unk_7c4: u32, /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). pub(crate) unk_7c8: u32, @@ -1285,7 +1352,10 @@ pub(crate) struct RegionC { pub(crate) compute_watchdog: u32, /// Required: 1. pub(crate) unk_9d0: u32, - pub(crate) unk_9d4: [u8; 0x449], + pub(crate) unk_9d4: [u8; 0xc], + /// See [`RegionCConfig::unk_9e0`]. + pub(crate) unk_9e0: [u32; 8], + pub(crate) unk_a00: [u8; 0x41d], /// Required: 1. pub(crate) unk_e1d: u32, pub(crate) unk_e21: [u8; 0x18], @@ -1319,28 +1389,28 @@ impl RegionC { pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); /// Initializes region C. - pub(crate) fn init(&mut self) { + pub(crate) fn init(&mut self, cfg: &RegionCConfig) { clear(self); self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; self.unk_24 = 0xbb8; self.idle_power_off = 1; self.unk_34 = 1; self.unk_3c = 0x78; - self.unk_54 = 0xffff; + self.unk_54 = cfg.unk_54; self.unk_56 = 0x28; self.unk_58 = 0xffff; self.unk_5e = 1; self.unk_62 = 1; self.unk_66 = 3; self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; - self.unk_78 = [1; 3]; - self.unk_84 = 0x254e; + self.unk_78 = cfg.unk_78; + self.max_power_mw = cfg.max_power_mw; self.unk_88 = 0x3e8; self.unk_8c = 0x3e8; self.unk_98 = 0x3e8; - self.unk_9c = 0x3000_001d; - self.unk_a0 = 0x3100_0002; - self.unk_7c4 = 0x7d0; + self.unk_9c = cfg.unk_9c; + self.unk_a0 = cfg.unk_a0; + self.unk_7c4 = cfg.unk_7c4; self.unk_7c8 = 0x3f80_0000; self.unk_7cc = 0x3f4c_cccd; self.unk_7d0 = 0x3e4c_cccd; @@ -1364,6 +1434,7 @@ impl RegionC { self.unk_9c8 = 0x28; self.compute_watchdog = 50; self.unk_9d0 = 1; + self.unk_9e0 = cfg.unk_9e0; self.unk_e1d = 1; self.unk_e39 = 3; self.unk_e3d = 1; @@ -1411,7 +1482,7 @@ impl StatusBlock { /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; /// Offset of the firmware-control ring in its page. -const FWCTL_RING: u64 = 0x40; +pub(crate) const FWCTL_RING: u64 = 0x40; /// One firmware-written recovery-info entry. The host reads each word separately after the /// recovery handshake reaches state 3; it never copies the live table as a structure. diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs new file mode 100644 index 0000000000000..c237896a49fb1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). +//! +//! The M4 Pro (T6040) runs this build. It has the interface version of the A18 Pro's +//! 26A428 build and the same objects, with these differences: +//! +//! - [`HwData25G83`]: the A18 Pro's hardware data up to `0x2554`, eight bytes inserted before +//! `0x2570` and four more before `0x2614`, and the CS and AFR performance tables; +//! - [`PowerConfig25G83`]: the power configuration with two blocks removed (`0xa8` and `0xa0` +//! bytes), at bundle `+0x5380`, followed by the auxiliary data; +//! - region C and the power state ([`POWER_STATE_WORDS`]) with other values; +//! - one firmware instance carrying both roles, with a larger status B ([`private`]), five +//! separately allocated views and a firmware area ([`kernel_window`]). +//! +//! The constant contents were derived from the G16 encoders of Gravity Linux, which run this +//! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. + +use super::{ + fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, RegisterMapEntry, + Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, COMPLETION_RING_INDEX, PERF_STATES, + REGISTER_MAP_SLOTS, +}; +use kernel::prelude::*; + +/// Kernel-window objects whose placement differs from the A18 Pro's, as offsets in the upper +/// kernel window. +pub(crate) mod kernel_window { + /// The power-state object named by the root (`0x6a8` bytes). + pub(crate) const POWER_STATE: u64 = 0x17_0000; + /// The five views named by the main configuration. + pub(crate) const VIEWS: [u64; 5] = [0x41_4000, 0x40_0000, 0x40_4000, 0x40_8000, 0x40_c000]; + /// The firmware area named twice by the main configuration (`0x88` bytes). + pub(crate) const FIRMWARE_AREA: u64 = 0x41_0000; + /// The AON temperature-source table named by the power configuration. + pub(crate) const AON_TABLE: u64 = 0xc050_0000; +} + +/// Offsets in the private cluster. +pub(crate) mod private { + /// Size of the private cluster: the state grid at `+0`, status B and status A. + pub(crate) const SIZE: usize = 0xd4000; + /// The primary status B ([`super::super::PrimaryStatusB`] up to its power state). + pub(crate) const STATUS_B: usize = 0x1_0000; + /// The primary status A, after the `0xf180` bytes of status B. + pub(crate) const STATUS_A: usize = 0x1_f180; + /// The configuration named by the root; its status follows at `+0x80`. + pub(crate) const CONFIG: usize = 0xd_3440; + /// The zeroed page named by the main configuration's `unk_18`. + pub(crate) const MAIN_HEADER: usize = 0x8000; +} + +/// Bundle offset of the [`PowerConfig25G83`]. +pub(crate) const POWER_CONFIG: usize = 0x5380; +/// Bundle offset of the completion-ring descriptors in the [`HwData25G83`]. +pub(crate) const COMPLETION_DESCRIPTORS: usize = 0x2620; +/// Bundle offset of the QoS pointer in the [`HwData25G83`]. +pub(crate) const QOS_POINTER: usize = 0x26b8; + +/// Most states of a CS or AFR performance table. +const AUX_PERF_STATES: usize = 16; + +/// The CS or AFR performance table of the hardware data. +#[repr(C, packed)] +pub(crate) struct AuxPerfTable { + /// Highest state, twice. + pub(crate) max_state: [u32; 2], + /// Frequency of each state in MHz. + pub(crate) freq_mhz: [u32; AUX_PERF_STATES], + /// Voltage of each state in mV, in the first word of each pair. + pub(crate) mv: [[u32; 2]; AUX_PERF_STATES], + /// SRAM voltage of each state in mV, in the first word of each pair. + pub(crate) sram_mv: [[u32; 2]; AUX_PERF_STATES], +} + +// SAFETY: `AuxPerfTable` consists of integers and arrays of integers only. +unsafe impl Zeroable for AuxPerfTable {} + +static_assert!(size_of::() == 0x148); + +impl AuxPerfTable { + /// Fills the table from its operating points (one voltage cell each), raising each SRAM + /// voltage to `sram_min_mv`. + fn fill(&mut self, opps: &[Opp], sram_min_mv: u32) -> Result { + if !(2..=AUX_PERF_STATES).contains(&opps.len()) { + return Err(EINVAL); + } + self.max_state = [opps.len() as u32 - 1; 2]; + for (index, opp) in opps.iter().enumerate() { + if opp.cells != 1 { + return Err(EINVAL); + } + let mv = opp.volt_uv[0] / 1000; + self.freq_mhz[index] = u32::try_from(opp.freq_hz / 1_000_000)?; + self.mv[index][0] = mv; + self.sram_mv[index][0] = mv.max(sram_min_mv); + } + Ok(()) + } +} + +/// Inputs of the 25G83 hardware data in addition to [`HwDataArgs`]. +pub(crate) struct HwDataArgs25G83<'a> { + /// Enabled GPU cores. + pub(crate) active_cores: u32, + /// Operating points of the CS clock domain. + pub(crate) cs_opps: &'a [Opp], + /// Operating points of the AFR clock domain. + pub(crate) afr_opps: &'a [Opp], +} + +/// The hardware data of the 25G83 build, at the base of the bundle. +#[repr(C, packed)] +pub(crate) struct HwData25G83 { + /// Required: 0x6f_0000_0000. + pub(crate) unk_0: u64, + /// Required: 0xffc0_0000. + pub(crate) unk_8: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_10: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_18: u64, + /// Required: 0x2ff_ffff_8000. + pub(crate) unk_20: u64, + /// Required: 0xffff_fc21_8140_0000. + pub(crate) unk_28: u64, + pub(crate) unk_30: [u8; 0x610], + /// Register windows the firmware maps. + pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], + pub(crate) unk_e88: [u8; 8], + /// Chip identifier. + pub(crate) chip_id: u32, + /// Required: 1. + pub(crate) unk_e94: u32, + /// Required: 1. + pub(crate) unk_e98: u32, + pub(crate) unk_e9c: [u8; 4], + /// Required: 1. + pub(crate) unk_ea0: u32, + /// Required: 1. + pub(crate) unk_ea4: u32, + pub(crate) unk_ea8: [u8; 0x10], + /// Required: 1. + pub(crate) unk_eb8: u32, + pub(crate) unk_ebc: [u8; 4], + /// Required: 1. + pub(crate) unk_ec0: u32, + pub(crate) unk_ec4: [u8; 4], + /// Required: 1. + pub(crate) unk_ec8: u32, + pub(crate) unk_ecc: [u8; 4], + /// Required: 0x5dc0 (24000). + pub(crate) unk_ed0: u32, + /// Required: 1. + pub(crate) unk_ed4: u32, + /// Power sample period in ms: 8. + pub(crate) unk_ed8: u32, + pub(crate) unk_edc: [u8; 4], + /// Required: 1 in every entry. + pub(crate) unk_ee0: [u32; 3], + pub(crate) unk_eec: [u8; 0x18], + /// Required: 0x1f (31). + pub(crate) unk_f04: u32, + pub(crate) unk_f08: [u8; 0x1c], + /// Required: 0x0700_2004. + pub(crate) unk_f24: u32, + /// Required: 6. + pub(crate) unk_f28: u32, + pub(crate) unk_f2c: [u8; 8], + /// Required: 1. + pub(crate) unk_f34: u32, + /// Required: 1. + pub(crate) unk_f38: u32, + pub(crate) unk_f3c: [u8; 0x10], + /// Required: 0x31 (49). + pub(crate) unk_f4c: u32, + pub(crate) unk_f50: [u8; 0x38], + /// Enabled GPU cores. + pub(crate) unk_f88: u32, + /// Required: 0xf (15). + pub(crate) unk_f8c: u32, + pub(crate) unk_f90: [u8; 0x28], + /// Required: 0x1c (28). + pub(crate) unk_fb8: u32, + /// Required: 4. + pub(crate) unk_fbc: u32, + /// Enabled GPU cores. + pub(crate) unk_fc0: u32, + /// Performance table A, one voltage lane per cluster. + pub(crate) perf_a: PerfGroup, + pub(crate) unk_16c8: [u8; 0x140], + /// Frequency of each state of table B in MHz. + pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], + pub(crate) unk_1834: [u8; 0x14], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_1848: [u32; 11], + pub(crate) unk_1874: [u8; 0x54], + /// Relative performance of each state of table A, in percent. + pub(crate) perf_a_relative: [u32; PERF_STATES], + pub(crate) unk_18f4: [u8; 0x14], + /// Relative performance of each state of table B, in percent. + pub(crate) perf_b_relative: [u32; PERF_STATES], + pub(crate) unk_1934: [u8; 0x94], + /// Operating point of each state of table A. + pub(crate) perf_a_states: [u32; PERF_STATES], + pub(crate) unk_19f4: [u8; 0x14], + /// Operating point of each state of table B. + pub(crate) perf_b_states: [u32; PERF_STATES], + pub(crate) unk_1a34: [u8; 0x14], + /// CS clock-domain performance table. + pub(crate) cs_perf: AuxPerfTable, + /// AFR clock-domain performance table. + pub(crate) afr_perf: AuxPerfTable, + /// Performance table B, one voltage lane per cluster. + pub(crate) perf_b: PerfGroup, + pub(crate) unk_23dc: [u8; 0x168], + /// Required: 5. + pub(crate) unk_2544: u32, + /// Required: 6. + pub(crate) unk_2548: u32, + /// Required: 3. + pub(crate) unk_254c: u32, + /// Required: 7. + pub(crate) unk_2550: u32, + /// Required: 7. + pub(crate) unk_2554: u32, + pub(crate) unk_2558: [u8; 0x18], + /// Enabled GPU cores. + pub(crate) unk_2570: u32, + pub(crate) unk_2574: [u8; 4], + /// Required: 1. + pub(crate) unk_2578: u32, + pub(crate) unk_257c: [u8; 0x1c], + /// Required: 0, 1, 1, 1. + pub(crate) unk_2598: [u32; 4], + /// Required: 0, 1, 1, 1. + pub(crate) unk_25a8: [u32; 4], + pub(crate) unk_25b8: [u8; 4], + /// Required: 0xffff_ffff in every entry, one per work channel. + pub(crate) unk_25bc: [u32; 12], + pub(crate) unk_25ec: [u8; 8], + /// Required: 2. + pub(crate) unk_25f4: u32, + pub(crate) unk_25f8: [u8; 4], + /// Required: 1. + pub(crate) unk_25fc: u32, + pub(crate) unk_2600: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2614: u32, + pub(crate) unk_2618: [u8; 4], + /// Required: 0x100. + pub(crate) unk_261c: u32, + /// Completion-ring descriptors. + pub(crate) completion_rings: [CompletionRingDescriptor; 4], + pub(crate) unk_26a0: [u8; 0x18], + /// QoS object, zeroed. + pub(crate) qos_va: u64, + /// Required: 1. + pub(crate) unk_26c0: u32, + /// Required: 1. + pub(crate) unk_26c4: u32, + pub(crate) unk_26c8: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_26f4: u32, + pub(crate) unk_26f8: [u8; 0x16bc], +} + +static_assert!(core::mem::offset_of!(HwData25G83, register_map) == 0x640); +static_assert!(core::mem::offset_of!(HwData25G83, chip_id) == 0xe90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_a) == 0xfc4); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b_freq_mhz) == 0x1808); +static_assert!(core::mem::offset_of!(HwData25G83, cs_perf) == 0x1a48); +static_assert!(core::mem::offset_of!(HwData25G83, afr_perf) == 0x1b90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b) == 0x1cd8); +static_assert!(core::mem::offset_of!(HwData25G83, unk_2570) == 0x2570); +static_assert!(core::mem::offset_of!(HwData25G83, completion_rings) == COMPLETION_DESCRIPTORS); +static_assert!(core::mem::offset_of!(HwData25G83, qos_va) == QOS_POINTER); +static_assert!(size_of::() == 0x3db4); +static_assert!(size_of::() <= POWER_CONFIG); + +// SAFETY: `HwData25G83` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwData25G83 {} + +impl HwData25G83 { + /// Initializes the hardware data. Table A and B carry one voltage lane per cluster. + pub(crate) fn init(&mut self, args: &HwDataArgs<'_>, ext: &HwDataArgs25G83<'_>) -> Result { + super::clear(self); + self.unk_0 = 0x6f_0000_0000; + self.unk_8 = 0xffc0_0000; + self.unk_10 = 0x10_0000_0000; + self.unk_18 = 0x10_0000_0000; + self.unk_20 = 0x2ff_ffff_8000; + self.unk_28 = 0xffff_fc21_8140_0000; + + let mut used = [false; REGISTER_MAP_SLOTS]; + for window in args.windows { + match used.get_mut(window.slot) { + Some(used) if !*used => *used = true, + _ => return Err(EINVAL), + } + self.register_map[window.slot] = RegisterMapEntry::window(window); + } + if !args.reserved_slots.is_empty() { + return Err(EINVAL); + } + + self.chip_id = args.chip_id; + self.unk_e94 = 1; + self.unk_e98 = 1; + self.unk_ea0 = 1; + self.unk_ea4 = 1; + self.unk_eb8 = 1; + self.unk_ec0 = 1; + self.unk_ec8 = 1; + self.unk_ed0 = 0x5dc0; + self.unk_ed4 = 1; + self.unk_ed8 = 8; + self.unk_ee0 = [1; 3]; + self.unk_f04 = 0x1f; + self.unk_f24 = 0x0700_2004; + self.unk_f28 = 6; + self.unk_f34 = 1; + self.unk_f38 = 1; + self.unk_f4c = 0x31; + self.unk_f88 = ext.active_cores; + self.unk_f8c = 0xf; + self.unk_fb8 = 0x1c; + self.unk_fbc = 4; + self.unk_fc0 = ext.active_cores; + + let perf = args.perf; + let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; + let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; + if states_a[0].cells < 2 { + return Err(EINVAL); + } + self.perf_a.fill(&states_a, 1); + self.perf_b.fill(&states_b, 1); + for (index, state) in states_b.iter().enumerate() { + self.perf_b_freq_mhz[index] = state.freq_mhz; + self.perf_a_states[index] = perf.states_a[index].into(); + self.perf_b_states[index] = perf.states_b[index].into(); + } + self.perf_a_relative = perf.relative_a; + self.perf_b_relative = perf.relative_b; + self.unk_1848 = [0x3f82_8f5c; 11]; + self.cs_perf.fill(ext.cs_opps, perf.sram_min_mv)?; + self.afr_perf.fill(ext.afr_opps, perf.sram_min_mv)?; + + self.unk_2544 = 5; + self.unk_2548 = 6; + self.unk_254c = 3; + self.unk_2550 = 7; + self.unk_2554 = 7; + self.unk_2570 = ext.active_cores; + self.unk_2578 = 1; + self.unk_2598 = [0, 1, 1, 1]; + self.unk_25a8 = [0, 1, 1, 1]; + self.unk_25bc = [u32::MAX; 12]; + self.unk_25f4 = 2; + self.unk_25fc = 1; + self.unk_2614 = 1; + self.unk_261c = 0x100; + for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { + self.completion_rings[index as usize] = CompletionRingDescriptor { + low_va: ring.low_va, + high_va: ring.high_va, + entries: COMPLETION_RING_ENTRIES, + entry_size: COMPLETION_RING_ENTRY_SIZE, + index, + unk_1c: 0, + }; + } + self.qos_va = args.qos_va; + self.unk_26c0 = 1; + self.unk_26c4 = 1; + self.unk_26f4 = 1; + Ok(()) + } +} + +/// Power configuration control period, in 24 MHz clocks (8 ms). +const PERIOD_CLOCKS: u32 = 200_000; +/// Highest GPU power, in mW. +pub(crate) const MAX_POWER_MW: u32 = 69_116; +/// [`MAX_POWER_MW`] as an IEEE-754 single. +const MAX_POWER_MW_F32: u32 = 0x4786_fe00; +/// Temperature sensors read by the die and MTR temperature limiters. +const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; + +/// Leakage coefficients, IEEE-754 singles: three arrays of sixteen clusters, then four arrays +/// of two dies. Not found in the ADT; constant for the SoC. +const LEAKAGE: [u32; 56] = { + let mut words = [0; 56]; + words[0] = 0x44f0_3000; // 1921.5 + words[1] = 0x44f3_0000; // 1944.0 + words[32] = 0x4373_0000; // 243.0 + words[33] = 0x4373_0000; // 243.0 + words[52] = 0x437b_0000; // 251.0 + words[54] = 0x43bd_c000; // 379.5 + words +}; +/// Offset of [`LEAKAGE`] in the power configuration. +const LEAKAGE_OFFSET: usize = 0x642c; +/// Offset in the power configuration of the GPU address of the configuration's status (the +/// configuration named by the root, `+0x80`). +const CONFIG_STATUS_POINTER: usize = 0x3b50; +/// Offset in the power configuration of the GPU address of the AON table. +const AON_POINTER: usize = 0x6318; +/// Offset in the power configuration of a random word the firmware expects. +const ENTROPY: usize = 0x63d4; + +/// Nonzero words of the power configuration and the auxiliary data that follows it, offsets +/// from the start of the power configuration. +const POWER_CONFIG_WORDS: Words = &[ + (0x4, PERIOD_CLOCKS, 2), + (0x10, 0x4, 1), + (0x14, 0x3f800000, 1), + (0x2c, 0x1, 2), + (0x40, 0x64, 1), + (0x44, 0x1, 1), + (0x48, 0x3e8, 1), + (0x50, 0x64, 1), + (0x60, 0x4, 1), + (0x64, 0x3f800000, 1), + (0x6c, 0x1, 2), + (0x80, 0x64, 1), + (0x84, 0x1, 1), + (0x88, 0x3e8, 1), + (0x90, 0x64, 1), + (0xc0, 0x3f828f5c, 11), + (0x9b8, 0x271, 1), + (0x9c4, 0x3f7f2e9f, 1), + (0x9cc, 0x3b516154, 1), + (0x9d4, 0x3ca59586, 1), + (0x9e0, MAX_POWER_MW_F32, 1), + (0x9e4, 0x40a90fdb, 1), + (0x9ec, 0xbc6d0cfe, 1), + (0x9f0, 0x64, 1), + (0x9f4, 0x3e8, 2), + (0xa04, MAX_POWER_MW, 1), + (0xa38, 0x3e8, 1), + (0xa70, 0x8, 1), + (0xa7c, 0x3f000000, 1), + (0xa84, 0x3f000000, 1), + (0xa8c, 0x3e3bd7b3, 1), + (0xa98, 0x47800000, 1), + (0xa9c, 0x3ecf2e49, 1), + (0xaa8, 0x64, 1), + (0xaac, 0x3e8, 1), + (0xabc, MAX_POWER_MW, 1), + (0xac8, 0x10, 1), + (0xad0, 0x5dc00, 1), + (0xb28, 0x50, 1), + (0xb30, 0x55, 1), + (0xb34, 0x32, 1), + (0xb38, 0x6, 1), + (0xb40, 0x6, 1), + (0xb44, 0x1, 1), + (0xb4c, 0x3f4ccccd, 1), + (0xb50, 0x3f7eb852, 1), + (0xb54, 0x3e4ccccd, 1), + (0xb58, 0x3ba3d70a, 1), + (0xb5c, 0x3f64c986, 2), + (0xb68, 0x42be0000, 1), + (0xb6c, 0x403ea7f0, 2), + (0xb74, 0x41179436, 1), + (0xb78, 0x64, 1), + (0xb7c, 0x3e8, 1), + (0xb80, 0x64, 1), + (0xb8c, 0x50, 1), + (0xbc0, 0x64, 1), + (0x1bec, 0xff000000, 1), + (0x1bf0, 0xff7fff7f, 255), + (0x1fec, 0x7fff7f, 1), + (0x21f0, 0x47800000, 1), + (0x21f4, 0x42c80000, 1), + (0x21f8, 0x447a0000, 1), + (0x21fc, 0x62a, 1), + (0x2210, 0x64, 1), + (0x2218, 0x42c80000, 1), + (0x221c, 0x3e8, 1), + (0x2224, 0x3f4ccccd, 1), + (0x2228, 0x3e4ccccd, 1), + (0x2268, TEMP_SENSOR_MASK, 1), + (0x2270, 0x1f40, 1), + (0x2278, 0x4, 1), + (0x228c, 0x3f800000, 1), + (0x2294, 0x414eeeee, 1), + (0x22a0, 0x47800000, 1), + (0x22a4, 0x42140000, 1), + (0x22b0, 0x64, 1), + (0x22b4, 0x3e8, 2), + (0x22c4, 0x2af8, 1), + (0x22f8, 0x3e8, 1), + (0x2728, 0xb8520000, 1), + (0x272c, 0x3f7e, 1), + (0x2730, 0xd70a0000, 1), + (0x2734, 0x3ba3, 1), + (0x273c, 0x43c8, 1), + (0x2748, 0x4780, 1), + (0x274c, 0x42c8, 1), + (0x2754, 0x384bac8, 1), + (0x2758, 0x3e80000, 2), + (0x2764, 0x80000000, 1), + (0x2768, 0x26ac453b, 1), + (0x279c, 0x3e80000, 1), + (0x27b8, 0xc80000, 1), + (0x2870, 0x3e8, 1), + (0x28c0, 0x1, 1), + (0x28c8, MAX_POWER_MW, 3), + (0x28e0, 0x3f800000, 1), + (0x28e4, 0x4, 1), + (0x28e8, 0x8, 1), + (0x28ec, 0x2ee00, 1), + (0x29bc, 0x7c, 1), + (0x29c8, 0x3f77bdef, 1), + (0x29d0, 0x3d042108, 1), + (0x29d8, 0x3d25bea1, 1), + (0x29e4, 0x47800000, 1), + (0x29e8, 0x3f4f2e49, 1), + (0x29f4, 0x28, 1), + (0x29f8, 0x3e8, 2), + (0x2a04, MAX_POWER_MW_F32, 1), + (0x2a08, MAX_POWER_MW, 1), + (0x2a14, 0xf8, 1), + (0x2a1c, 0x5ad200, 1), + (0x2a3c, 0x3e8, 1), + (0x2b4c, 0x32, 1), + (0x2b50, 0x1, 1), + (0x2b58, 0x3f638e39, 1), + (0x2b5c, 0x3f2aaaab, 1), + (0x2b60, 0x3de38e39, 1), + (0x2b64, 0x3eaaaaab, 1), + (0x2b68, 0xbecccccd, 2), + (0x2b74, 0x47800000, 1), + (0x2b78, 0xc0a00000, 2), + (0x2b84, 0x64, 1), + (0x2b88, 0x3e8, 1), + (0x2b8c, 0x64, 1), + (0x2b94, 0x45fa0000, 1), + (0x2b98, 0x578, 1), + (0x2ba4, 0x48, 1), + (0x2ba8, 0x18, 1), + (0x2bac, 0x1a5e00, 1), + (0x2bb4, 0x8ca00, 1), + (0x2bc0, 0x47800000, 1), + (0x2be8, 0x1f40, 1), + (0x2bec, 0x4, 1), + (0x2bf0, 0x9c4, 1), + (0x2bf4, 0x258, 1), + (0x2bf8, 0x2, 1), + (0x2bfc, 0x4, 1), + (0x2c00, 0x32, 1), + (0x2c30, 0x47800000, 1), + (0x2c40, 0x64, 1), + (0x2c44, 0x3e8, 1), + (0x2e2c, 0x4, 1), + (0x2e38, 0x3f780000, 1), + (0x2e40, 0x3d000000, 1), + (0x2e48, 0x40a00000, 1), + (0x2e54, 0x47800000, 1), + (0x2e58, 0x41700000, 1), + (0x2e64, 0x384, 1), + (0x2e68, 0x3e8, 2), + (0x2e78, 0x3e8, 1), + (0x2eac, 0x3e8, 1), + (0x2eec, 0x64, 1), + (0x2f08, TEMP_SENSOR_MASK, 1), + (0x2f10, 0x7d, 1), + (0x2f14, 0x1, 1), + (0x3b58, TEMP_SENSOR_MASK, 1), + (0x3b60, 0x1, 1), + (0x5b64, 0x40a00000, 1), + (0x6320, u32::from_be_bytes(*b"g0TR"), 1), + (0x6324, u32::from_be_bytes(*b"g1TR"), 1), + (0x6360, 0x3, 1), + (0x6368, u32::from_be_bytes(*b"r0TR"), 1), + (0x6370, 0x1, 1), + (0x6420, 0x1, 1), +]; + +/// The power configuration and auxiliary data of the 25G83 build, at bundle +/// [`POWER_CONFIG`]. The main configuration names it. +#[repr(C)] +pub(crate) struct PowerConfig25G83(pub(crate) [u8; 0x6510]); + +// SAFETY: `PowerConfig25G83` is a byte array. +unsafe impl Zeroable for PowerConfig25G83 {} + +static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); + +impl PowerConfig25G83 { + /// Pointer and random words, which [`Self::hash`] leaves out. + const RUNTIME: [(usize, usize); 3] = + [(CONFIG_STATUS_POINTER, 8), (AON_POINTER, 8), (ENTROPY, 4)]; + + /// Initializes the power configuration. + pub(crate) fn init(&mut self, config_va: u64, aon_va: u64, entropy: u32) { + fill_words(&mut self.0, POWER_CONFIG_WORDS); + for (index, word) in LEAKAGE.iter().enumerate() { + let at = LEAKAGE_OFFSET + 4 * index; + self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); + } + self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] + .copy_from_slice(&(config_va + 0x80).to_le_bytes()); + self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); + self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); + } + + /// Hash of the contents without the pointers and the random word. + pub(crate) fn hash(&self) -> u64 { + hash_without(&self.0, &Self::RUNTIME) + } +} + +/// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU +/// cluster 1 and SoC sensors. +pub(crate) const AON_TABLE: [u32; 9] = [ + u32::from_be_bytes(*b"g0TR"), + 0x006b_0001, + 0x0000_1401, + u32::from_be_bytes(*b"g1TR"), + 0x00b5_0001, + 0x0000_1401, + u32::from_be_bytes(*b"r0TR"), + 0x00c1_0001, + 0x0000_1401, +]; + +/// Nonzero words of the power-state object. The fast-die limiter reads its target +/// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero +/// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. +pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; + +impl HwData25G83 { + /// What [`Self::hash`] leaves out: the runtime words (completion descriptors, QoS pointer) + /// and what comes from the device: the performance tables built from its operating points, + /// which are binned per device, and its enabled core count. + const UNHASHED: [(usize, usize); 10] = [ + (COMPLETION_DESCRIPTORS, 0x80), + (QOS_POINTER, 8), + (core::mem::offset_of!(Self, unk_f88), size_of::()), + (core::mem::offset_of!(Self, unk_fc0), size_of::()), + (core::mem::offset_of!(Self, perf_a), size_of::()), + (core::mem::offset_of!(Self, perf_b_freq_mhz), size_of::<[u32; PERF_STATES]>()), + (core::mem::offset_of!(Self, cs_perf), size_of::()), + (core::mem::offset_of!(Self, afr_perf), size_of::()), + (core::mem::offset_of!(Self, perf_b), size_of::()), + (core::mem::offset_of!(Self, unk_2570), size_of::()), + ]; + + /// Hash of the constant contents (everything but [`Self::UNHASHED`]). + pub(crate) fn hash(&self) -> u64 { + // SAFETY: `HwData25G83` consists of integers only and has no padding. + let bytes = unsafe { + core::slice::from_raw_parts((self as *const Self).cast::(), size_of::()) + }; + hash_without(bytes, &Self::UNHASHED) + } +} + +/// FNV-1a 64 of `bytes` with the `(offset, length)` ranges in `skip` left out. +pub(crate) fn hash_without(bytes: &[u8], skip: &[(usize, usize)]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for (offset, byte) in bytes.iter().enumerate() { + if skip.iter().any(|&(at, len)| (at..at + len).contains(&offset)) { + continue; + } + hash = (hash ^ u64::from(*byte)).wrapping_mul(0x100_0000_01b3); + } + hash +} + +/// Hashes ([`hash_without`]) of the hardware data, power configuration and region C that the +/// Gravity Linux G16 encoders produce for the J616s, which this build's objects must equal. +pub(crate) mod expected { + pub(crate) const HW_DATA: u64 = 0xfd3b_a26f_570f_57cd; + pub(crate) const POWER_CONFIG: u64 = 0xb669_5b84_2ad2_36fe; + pub(crate) const REGION_C: u64 = 0xd2db_fa6b_8976_47b4; +} diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index e07fbfb485a10..4f3a70671eb87 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -234,7 +234,7 @@ impl McacheRange { return Err(EINVAL); } Ok(Self { - start: start | (context as u64) << Self::CONTEXT_SHIFT | Self::UNK_START, + start: start | u64::from(context) << Self::CONTEXT_SHIFT | Self::UNK_START, end: end | Self::UNK_END | 1 << (Self::CLASS_SHIFT + class as u32), }) } @@ -297,7 +297,7 @@ impl RegisterArrayBinding { return Err(EINVAL); } Ok(Self { - word: packed_va(array_va)? | (checkpoint as u64) << Self::CHECKPOINT_SHIFT, + word: packed_va(array_va)? | u64::from(checkpoint) << Self::CHECKPOINT_SHIFT, }) } @@ -332,7 +332,7 @@ impl McacheTable { fn word(self) -> u64 { (self.va >> PACKED_VA_SHIFT) << Self::VA_SHIFT - | ((self.count - 1) as u64) << Self::COUNT_SHIFT + | u64::from(self.count - 1) << Self::COUNT_SHIFT } } @@ -444,8 +444,8 @@ impl KickEntry { } let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT - | (args.qid as u64) << Self::QID_SHIFT - | (args.priority as u64) << Self::PRIORITY_SHIFT; + | u64::from(args.qid) << Self::QID_SHIFT + | u64::from(args.priority) << Self::PRIORITY_SHIFT; if args.scs_enabled { header |= Self::SCS_ENABLED; } @@ -478,9 +478,9 @@ impl KickEntry { mcache, descriptor_va: args.descriptor_va, queue_va: args.queue_va, - dependency_control: mask as u64 - | (args.qos.slot as u64) << Self::QOS_SLOT_SHIFT - | (args.qos.class as u64) << Self::QOS_CLASS_SHIFT + dependency_control: u64::from(mask) + | u64::from(args.qos.slot) << Self::QOS_SLOT_SHIFT + | u64::from(args.qos.class) << Self::QOS_CLASS_SHIFT | Self::UNK_DEPENDENCY_CONTROL, dependencies, event_mask: args.event_mask, diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index b6bb6aaea860f..1cb46fce2d602 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -23,6 +23,9 @@ pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; pub(crate) mod qos; +pub(crate) mod variant; + +pub(crate) use variant::Variant; /// Clears a firmware structure in place. /// @@ -30,7 +33,7 @@ pub(crate) mod qos; pub(crate) fn clear(object: &mut T) { // SAFETY: `T: Zeroable` guarantees that all-zero bytes are a valid `T`, // and `object` is a valid, exclusive reference to one `T`. - unsafe { core::ptr::write_bytes(object as *mut T, 0, 1) }; + unsafe { core::ptr::write_bytes(core::ptr::from_mut(object), 0, 1) }; } /// Returns `va + offset`, or `EINVAL` if the sum overflows. diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 48b110a4193e3..5708a2251a062 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -20,6 +20,8 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::prelude::*; +use super::variant::PointerLayout; + use super::kick::{KickTimestamp, QID_MAX}; /// Firmware scheduling profile of a queue. @@ -94,6 +96,10 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, + /// [`FragmentBarrier`] (25G83 firmware). + FragmentBarrier = 4, + /// [`InitBufferManager`] (25G83 firmware). + InitBufferManager = 5, } /// The engine that executes a queue's work. @@ -115,26 +121,29 @@ pub(crate) enum DataMaster { /// /// - creation: the whole block, [`PointerBlock::new`], before the queue is /// first published; -/// - publication: `producer`, with one aligned 32-bit store per publication +/// - publication: the producer, with one aligned 32-bit store per publication /// step, after a barrier that orders the work pointers and commands before /// it; /// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. +/// +/// Where the producer and the recovery target live depends on the firmware +/// build ([`PointerLayout`]). #[repr(C)] #[derive(Debug)] pub(crate) struct PointerBlock { /// Item-ring position up to which the firmware has finished commands. - /// Free ring space is measured from it, so it bounds `producer`. + /// Free ring space is measured from it, so it bounds the producer. pub(crate) done: AtomicU32, pub(crate) unk_04: [u32; 11], /// Written by the firmware. The host treats the queue as idle only when - /// this cursor, `done` and `producer` are equal. + /// this cursor, `done` and the producer are equal. pub(crate) unk_30: AtomicU32, pub(crate) unk_34: [u32; 3], - /// Item-ring position after the last work pointer the host published. - pub(crate) producer: AtomicU32, + /// 26A428: item-ring position after the last work pointer the host published. + word_40: AtomicU32, pub(crate) unk_44: [u32; 3], - /// On recovery, the host copies producer here before acknowledging resume. - pub(crate) recovery_target: u32, + /// 26A428: on recovery, the host copies the producer here before acknowledging resume. + word_50: AtomicU32, pub(crate) unk_54: [u32; 3], /// Number of entries in the item ring. pub(crate) entries: u32, @@ -143,29 +152,62 @@ pub(crate) struct PointerBlock { static_assert!(size_of::() == 0x80); static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); -static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_40) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_50) == 0x50); static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); // SAFETY: PointerBlock contains only integers and integer atomics. unsafe impl Zeroable for PointerBlock {} impl PointerBlock { - pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); - pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); + /// Offsets of the two host words a [`PointerLayout`] may name. + const WORD_40: usize = core::mem::offset_of!(Self, word_40); + const WORD_50: usize = core::mem::offset_of!(Self, word_50); + + /// Whether `offset` is one of the host words a [`PointerLayout`] may name. + const fn is_host_word(offset: usize) -> bool { + offset == Self::WORD_40 || offset == Self::WORD_50 + } + + /// Whether `layout` names host words of the block, and two different ones when it has a + /// recovery target. Every firmware variant's layout is checked at build time. + pub(crate) const fn valid_layout(layout: &PointerLayout) -> bool { + match layout.recovery_target { + None => Self::is_host_word(layout.producer), + Some(target) => { + Self::is_host_word(layout.producer) + && Self::is_host_word(target) + && target != layout.producer + } + } + } /// Returns the cursors of an empty item ring. - pub(crate) const fn new() -> Self { - Self { + pub(crate) fn new(layout: &PointerLayout) -> Result { + let block = Self { done: AtomicU32::new(0), unk_04: [0; 11], unk_30: AtomicU32::new(0), unk_34: [0; 3], - producer: AtomicU32::new(0), + word_40: AtomicU32::new(0), unk_44: [0; 3], - recovery_target: u32::MAX, + word_50: AtomicU32::new(0), unk_54: [0; 3], entries: ITEM_RING_ENTRIES, unk_64: [0; 7], + }; + if let Some(target) = layout.recovery_target { + block.word(target)?.store(u32::MAX, Ordering::Relaxed); + } + Ok(block) + } + + /// The host word at `offset`, which must be one a [`PointerLayout`] may name. + fn word(&self, offset: usize) -> Result<&AtomicU32> { + match offset { + Self::WORD_40 => Ok(&self.word_40), + Self::WORD_50 => Ok(&self.word_50), + _ => Err(EINVAL), } } @@ -174,12 +216,14 @@ impl PointerBlock { /// Used when a render queue pair that was kicked before is handed to a new /// owner, with `base` the position the host tracks for the new owner's /// queues. Only valid while the firmware does not run the queue, with all - /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this + /// three cursors zero. Writes `done`, `unk_30` and the producer, in this /// order, with single stores; the caller issues a full barrier afterwards. - pub(crate) fn rebase(&self, base: u32) { + pub(crate) fn rebase(&self, layout: &PointerLayout, base: u32) -> Result { + let producer = self.word(layout.producer)?; self.done.store(base, Ordering::Relaxed); self.unk_30.store(base, Ordering::Relaxed); - self.producer.store(base, Ordering::Relaxed); + producer.store(base, Ordering::Relaxed); + Ok(()) } } @@ -271,12 +315,12 @@ impl QueueRecord { job_list_va: args.job_list_va, unk_18: [0; 0xc], unk_24: u32::MAX, - priority: [args.policy.priority() as u32; 2], + priority: [u32::from(args.policy.priority()); 2], unk_30: [0; 6], unk_36: u16::MAX, unk_38: 0, unk_3c: 0, - policy: args.policy.policy() as u32, + policy: u32::from(args.policy.policy()), unk_44: u32::MAX, owner_pid: args.owner_pid, unk_4c: [0; 0x50], @@ -307,12 +351,12 @@ impl QueueRecord { /// Replaces the priority and policy words of an idle queue's record when they differ /// from `policy`. pub(crate) fn set_policy(&mut self, policy: Policy) { - let priority = [policy.priority() as u32; 2]; + let priority = [u32::from(policy.priority()); 2]; if { self.priority } != priority { self.priority = priority; } - if { self.policy } != policy.policy() as u32 { - self.policy = policy.policy() as u32; + if { self.policy } != u32::from(policy.policy()) { + self.policy = u32::from(policy.policy()); } } } @@ -480,7 +524,7 @@ impl QueueConfig { binding.pb_va, binding.pb_token, binding.pb_slot, - binding.fragment_qid as u32, + u32::from(binding.fragment_qid), ) } QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), @@ -491,14 +535,14 @@ impl QueueConfig { unk_04: 0, kick_ring_va: args.kick_ring_va, kick_ring_fw_va: args.kick_ring_fw_va, - qid: args.qid as u16, - install: args.install as u16, + qid: u16::from(args.qid), + install: u16::from(args.install), unk_1c: 0, - priority: args.policy.priority() as u16, + priority: u16::from(args.policy.priority()), unk_20: 0, data_master: data_master as u16, unk_24: 0, - completion_seed_valid: args.completion_seed.is_some() as u32, + completion_seed_valid: u32::from(args.completion_seed.is_some()), completion_seed: args.completion_seed.unwrap_or(0), context_id: args.context_id, unk_34: 0, @@ -506,13 +550,13 @@ impl QueueConfig { free_list_generation: args.free_list.generation, free_list_slot: args.free_list.slot, scheduler_va: args.scheduler_va, - qos_update: args.qos_update as u32, - qos_slot: args.qos_slot as u16, + qos_update: u32::from(args.qos_update), + qos_slot: u16::from(args.qos_slot), unk_58: 0, owner_pid: args.owner_pid, policy: args.policy.policy(), unk_60: 0, - context_update: args.context_update as u16, + context_update: u16::from(args.context_update), unk_64: 0, unk_66: Self::UNK_66, unk_68: [0; 6], @@ -525,6 +569,94 @@ impl QueueConfig { } } +/// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work +/// waits for the paired tiling stamp, in addition to the kick-entry dependency. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FragmentBarrier { + /// [`CommandTag::FragmentBarrier`]. + pub(crate) tag: u32, + /// GPU address of the tiling queue's completion stamp word. + pub(crate) stamp_va: u64, + /// Timestamp of the tiling kick. + pub(crate) tiling_kick: u64, + /// QID of the tiling queue. + pub(crate) tiling_qid: u32, + /// Tiling stamp value waited for. + pub(crate) wait: u32, + pub(crate) unk_1c: [u8; 8], + /// Event of the wait: the tiling QID. + pub(crate) event: u32, + /// Fragment stamp value. + pub(crate) stamp_self: u32, + /// Current object ID of the fragment stage. + pub(crate) fragment_id: u64, + pub(crate) unk_34: u32, + pub(crate) unk_38: [u8; 0x10], +} + +static_assert!(size_of::() == 0x48); +static_assert!(core::mem::offset_of!(FragmentBarrier, wait) == 0x18); +static_assert!(core::mem::offset_of!(FragmentBarrier, event) == 0x24); +static_assert!(core::mem::offset_of!(FragmentBarrier, fragment_id) == 0x2c); + +impl FragmentBarrier { + /// The barrier of a pass whose tiling stage, kicked at `tiling_kick` on `tiling_qid`, + /// signals `stamp_value` at `stamp_va`. + pub(crate) fn new( + stamp_va: u64, + tiling_kick: KickTimestamp, + tiling_qid: u32, + stamp_value: u32, + fragment_id: u64, + ) -> Self { + Self { + tag: CommandTag::FragmentBarrier as u32, + stamp_va, + tiling_kick: tiling_kick.get(), + tiling_qid, + wait: stamp_value, + unk_1c: [0; 8], + event: tiling_qid, + stamp_self: stamp_value, + fragment_id, + unk_34: 1, + unk_38: [0; 0x10], + } + } +} + +/// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling +/// descriptor that uses a parameter buffer. The block count is read from the manager. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct InitBufferManager { + /// [`CommandTag::InitBufferManager`]. + pub(crate) tag: u32, + /// UAT context of the pass. + pub(crate) context_id: u32, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// Tiling stamp value of the pass. + pub(crate) stamp_value: u32, +} + +static_assert!(size_of::() == 0x18); + +impl InitBufferManager { + pub(crate) fn new(context_id: u32, pb_slot: u32, pb_va: u64, stamp_value: u32) -> Self { + Self { + tag: CommandTag::InitBufferManager as u32, + context_id, + pb_slot, + pb_va, + stamp_value, + } + } +} + /// Announcement of a newly written kick entry of a queue (tag 14). /// /// A record slot larger than this structure must be zero beyond it. @@ -564,7 +696,7 @@ impl KickAnnounce { } Ok(Self { tag: CommandTag::KickAnnounce as u32, - qid: qid as u16, + qid: u16::from(qid), count: 1, priority, timestamp: timestamp.get(), @@ -603,7 +735,7 @@ impl KickPredecessor { } Ok(Self { tag: CommandTag::KickPredecessor as u32, - qid: qid as u16, + qid: u16::from(qid), previous: previous.get(), unk_0e: Self::UNK_0E, }) diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 85a2385b421b8..0e270d40392a8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -48,10 +48,53 @@ pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; const USER_BASE: u64 = 0x10_0000_0000; /// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. const VDM_OFFSET_LIMIT: u64 = 1 << 39; -/// The deflake buffer holds three sub-buffers: at its start and at these -/// offsets. -const DEFLAKE_20: u64 = 0x20; -const DEFLAKE_2A0: u64 = 0x2a0; +/// The deflake (context) buffer holds three sub-buffers: at its start and at two offsets +/// that follow the control words of each cluster. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Deflake { + /// Offsets of the second and third sub-buffers. + pub(crate) sub: [u64; 2], + /// The host writes the viewport words at `+0x900`; with more clusters that offset lies in + /// the third sub-buffer and the buffer is left to the GPU. + pub(crate) viewport: bool, +} + +impl Deflake { + /// One cluster (A18 Pro). + pub(crate) const ONE_CLUSTER: Deflake = Deflake { + sub: [0x20, 0x2a0], + viewport: true, + }; + /// Two clusters (M4 Pro). + pub(crate) const TWO_CLUSTERS: Deflake = Deflake { + sub: [0xa0, 0x820], + viewport: false, + }; +} + +/// The register programs of a firmware build's render descriptors. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum RegisterPrograms { + /// The A18 Pro firmware's (26A428). + G17P, + /// The 25G83 build's: other tiling and fragment arrays, another tile-state base, and the + /// tiler parameter sub-buffers in the TPC object. + V25G83, +} + +/// The 25G83 tiling and main fragment arrays. +const TA_REGISTERS_25G83: usize = 83; +const FRAGMENT_REGISTERS_25G83: usize = 91; +const TA_CHECKPOINTS_25G83: [u16; 3] = [81, 83, 83]; +const FRAGMENT_CHECKPOINTS_25G83: [u16; 3] = [87, 88, 88]; +/// Index of the free-list slot register (0x14308) in the 25G83 tiling array. +const TA_SLOT_INDEX_25G83: u8 = 79; +/// Index of the free-list slot register (0x14048) in the 25G83 fragment array. +const FRAGMENT_SLOT_INDEX_25G83: u8 = 84; +/// A 25G83 stage with shader-context scratch (ray queries) also writes the SCS enable 0x017d9 = 1, +/// which the compute program and both A18 Pro render programs carry for such work: first in the +/// tiling array, after 0x01739 in the fragment array. Every count and index after it moves by one. +const SCS_ENABLE: u32 = 0x017d9; const TA_REGISTERS: usize = 73; const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; @@ -104,6 +147,10 @@ const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; /// 32, the utile size bits of the tile-buffer configuration in bits 43:40, /// tiles in X less one from bit 44 and tiles in Y less one from bit 53. const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; +/// The 25G83 build's low tile-state bits, the ones the M4 Pro has run with. With the A18 Pro's, +/// a pass more than 80 tiles wide and with more than 4096 tiles never completes its fragment +/// stage (the 3456x2234 panel is 108 x 70 tiles). +const TILE_STATE_BASE_25G83: u64 = 0x36011 | 1 << 37; const TILE_STATE_LAYERED: u64 = 1 << 32; const TILE_STATE_UTILE_SHIFT: u32 = 28; const TILE_STATE_TILES_X_SHIFT: u32 = 44; @@ -262,6 +309,9 @@ pub(crate) struct RenderResources { pub(crate) tpc_va: u64, /// Size of the tail-pointer cache the target needs. pub(crate) tpc_size: u64, + /// GPU addresses of the tiler parameter sub-buffers (entry 0 is the TPC); zero for + /// an empty one and on other GPUs. + pub(crate) gtp_va: [u64; 8], /// GPU address of the deflake buffer. pub(crate) deflake_va: u64, /// GPU address of the tiling status word. @@ -314,6 +364,10 @@ pub(crate) struct RenderArgs<'a> { pub(crate) ta: RenderStage, /// Fragment stage. pub(crate) fragment: RenderStage, + /// Register programs of the firmware build. + pub(crate) programs: RegisterPrograms, + /// Deflake-buffer layout of the GPU. + pub(crate) deflake: Deflake, } impl RenderArgs<'_> { @@ -357,12 +411,12 @@ impl Geometry { ) { return Err(EINVAL); } - let width = pass.width as u64; - let height = pass.height as u64; + let width = u64::from(pass.width); + let height = u64::from(pass.height); let tiles_x = width.div_ceil(TILE_SIZE); let tiles_y = height.div_ceil(TILE_SIZE); - let utiles_x = TILE_SIZE / pass.utile_width as u64; - let utiles_y = TILE_SIZE / pass.utile_height as u64; + let utiles_x = TILE_SIZE / u64::from(pass.utile_width); + let utiles_y = TILE_SIZE / u64::from(pass.utile_height); let utiles = utiles_x * utiles_y; let mtile_x = tiles_x.div_ceil(MACRO_TILES).next_multiple_of(4); let mtile_y = tiles_y.div_ceil(MACRO_TILES).next_multiple_of(4); @@ -411,7 +465,7 @@ impl PassWords { // The tile configuration carries the empty-tile request for every // target; the tiling layer mode only for layered ones. let mut tile_config = TILE_CONFIG_BASE; - let mut layer_mode = LAYER_MODE_BASE | (pass.layers as u64 - 1); + let mut layer_mode = LAYER_MODE_BASE | (u64::from(pass.layers) - 1); if layered { tile_config |= TILE_CONFIG_LAYERED; layer_mode |= LAYER_MODE_LAYERED; @@ -441,19 +495,19 @@ impl PassWords { } let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { true => spec, - false => spec & u32::MAX as u64, + false => spec & u64::from(u32::MAX), }; - let usc = |offset: u32| offset_va(pass.usc_base, (offset & !USC_OFFSET_TAG_MASK) as u64); + let usc = |offset: u32| offset_va(pass.usc_base, u64::from(offset & !USC_OFFSET_TAG_MASK)); Ok(Self { - utile_config: (pass.utile_width as u64 / 16) << UTILE_CONFIG_WIDTH_SHIFT - | (pass.utile_height as u64 / 16) << UTILE_CONFIG_HEIGHT_SHIFT - | pass.samples_log2 as u64, + utile_config: (u64::from(pass.utile_width) / 16) << UTILE_CONFIG_WIDTH_SHIFT + | (u64::from(pass.utile_height) / 16) << UTILE_CONFIG_HEIGHT_SHIFT + | u64::from(pass.samples_log2), tile_config, layer_mode, - tib_blocks: (pass.sample_size as u64 - * pass.utile_width as u64 - * pass.utile_height as u64 + tib_blocks: (u64::from(pass.sample_size) + * u64::from(pass.utile_width) + * u64::from(pass.utile_height) * samples) .div_ceil(TIB_BLOCK_SIZE), partial_zls_control, @@ -491,9 +545,10 @@ fn ta_registers( let layermeta = user_offset(r.layermeta_va)?; let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; let tpc = user_offset(r.tpc_va)?; - let deflake = user_offset32(r.deflake_va)? as u64; - let deflake_20 = user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)? as u64; - let deflake_2a0 = user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)? as u64; + let [deflake_20, deflake_2a0] = args.deflake.sub; + let deflake = u64::from(user_offset32(r.deflake_va)?); + let deflake_20 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_20)?)?); + let deflake_2a0 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_2a0)?)?); let vdm = user_offset(p.vdm_va)?; if vdm >= VDM_OFFSET_LIMIT { return Err(EINVAL); @@ -561,7 +616,7 @@ fn ta_registers( RegisterWrite::new(0x0d471, rt.scratch.tail_word()), RegisterWrite::new(0x1a0f1, rt.scratch.stack_word()), RegisterWrite::new(0x10799, rt.scratch.window_word()), - RegisterWrite::new(0x1c830, r.pb_slot as u64), + RegisterWrite::new(0x1c830, u64::from(r.pb_slot)), RegisterWrite::new(0x1ca30, r.pm_scratch), RegisterWrite::new(0x16c39, r.pm_scratch), RegisterWrite::new(0x1c910, r.pm_metric), @@ -573,7 +628,7 @@ fn ta_registers( RegisterWrite::new(0x10209, key), RegisterWrite::new(0x1c9f0, key), RegisterWrite::new(0x14320, key), - RegisterWrite::new(0x14308, r.free_list_slot as u64), + RegisterWrite::new(0x14308, u64::from(r.free_list_slot)), RegisterWrite::new(0x14318, r.ta_status_va | 1), RegisterWrite::new(0x01740, 1), RegisterWrite::new(0x1c880, deflake_2a0), @@ -581,6 +636,129 @@ fn ta_registers( ]) } +/// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage +/// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the +/// M4 Pro has run with. Returns the array and its used entry count. +fn ta_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; TA_REGISTERS_25G83 + 1], usize)> { + let a = ta_registers(args, g, w)?; + let r = args.resources; + let gtp = |index: usize| -> Result { + match r.gtp_va[index] { + 0 => Ok(0), + va => user_offset(va), + } + }; + let find = |reg: u32, nth: usize| -> Result { + a.iter() + .filter(|write| write.reg == reg) + .nth(nth) + .copied() + .ok_or(EINVAL) + }; + let gtp1 = gtp(1)?; + let gtp2 = gtp(2)?; + let gtp6 = gtp(6)? & !127; + let scs = usize::from(args.ray_query[0].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); TA_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut push = |write: RegisterWrite| { + out[n] = write; + n += 1; + }; + if scs != 0 { + push(find(SCS_ENABLE, 0)?); + } + for reg in [0x10141, 0x1c039, 0x1c9c8] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x1c041, gtp1)); + push(RegisterWrite::new(0x1c9d0, gtp1)); + for reg in [ + 0x1c0a1, 0x1c031, 0x1c9c0, 0x1c051, 0x1c061, 0x10149, 0x10139, 0x10111, 0x1c9b0, 0x10119, + 0x1c9b8, 0x1c958, 0x1c950, 0x1c930, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 0)?); + push(RegisterWrite::new(0x1c948, gtp(4)?)); + push(RegisterWrite::new(0x1c888, gtp(5)?)); + push(RegisterWrite::new(0x1c890, 0)); + push(RegisterWrite::new(0x1c918, 0)); + push(find(0x1c079, 0)?); + push(find(0x1c9d8, 0)?); + push(RegisterWrite::new(0x1c089, gtp2)); + push(RegisterWrite::new(0x1c9e0, gtp2)); + push(RegisterWrite::new(0x16c41, gtp6)); + push(RegisterWrite::new(0x1ca40, gtp6)); + push(RegisterWrite::new(0x1c9a8, 0)); + push(RegisterWrite::new(0x1c920, gtp(7)?)); + for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x10131, 0x88)); + for reg in [ + 0x10121, 0x10129, 0x101b9, 0x1c069, 0x1c071, 0x1c081, 0x1c0a9, 0x10171, 0x10169, 0x0a309, + 0x1c8e0, 0x1c8e8, + ] { + push(find(reg, 0)?); + } + push(find(0x1c898, 0)?); + for reg in [ + 0x101e1, 0x1c9e8, 0x1a099, 0x1a0a1, 0x1a069, 0x1a071, 0x1a0c9, 0x1a0d1, 0x101c9, 0x0d471, + 0x1a0f1, 0x10799, 0x1c830, 0x1ca30, 0x16c39, 0x1c910, 0x0a5a1, 0x0d419, 0x1ca10, 0x014a1, + 0x0a349, 0x10209, 0x1c9f0, 0x14320, 0x14308, 0x14318, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 1)?); + push(find(0x1c898, 1)?); + if n != TA_REGISTERS_25G83 + scs || out[TA_SLOT_INDEX_25G83 as usize + scs].reg != 0x14308 { + return Err(EINVAL); + } + Ok((out, n)) +} + +/// The 25G83 main fragment array: the A18 Pro's (without 0x17d9 unless the stage has SCS), with +/// 0x160b8 = 0 after the tile state and 0x160a8 = 0x20 before the closing 0x16068, as the M4 +/// Pro has run with. Returns the array and its used entry count. +fn fragment_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; FRAGMENT_REGISTERS_25G83 + 1], usize)> { + let a = fragment_registers(args, g, w); + let scs = usize::from(args.ray_query[1].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); FRAGMENT_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut tile_state = false; + for (index, write) in a.iter().enumerate() { + if write.reg == SCS_ENABLE && scs == 0 { + continue; + } + if index == a.len() - 1 { + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160a8, 0x20); + n += 1; + } + *out.get_mut(n).ok_or(EINVAL)? = *write; + n += 1; + if write.reg == 0x16068 && !tile_state { + tile_state = true; + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160b8, 0); + n += 1; + } + } + if n != FRAGMENT_REGISTERS_25G83 + scs + || out[FRAGMENT_SLOT_INDEX_25G83 as usize + scs].reg != 0x14048 + { + return Err(EINVAL); + } + Ok((out, n)) +} + fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, @@ -588,7 +766,12 @@ fn fragment_registers( ) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; - let mut tile_state = TILE_STATE_BASE + let tile_state_base = if args.programs == RegisterPrograms::V25G83 { + TILE_STATE_BASE_25G83 + } else { + TILE_STATE_BASE + }; + let mut tile_state = tile_state_base | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; @@ -609,8 +792,8 @@ fn fragment_registers( RegisterWrite::new(0x15381, w.eot_va), RegisterWrite::new(0x15369, w.bg_rsrc_spec), RegisterWrite::new(0x15371, w.bg_va), - RegisterWrite::new(0x15131, p.merge_upper_x as u64), - RegisterWrite::new(0x15139, p.merge_upper_y as u64), + RegisterWrite::new(0x15131, u64::from(p.merge_upper_x)), + RegisterWrite::new(0x15139, u64::from(p.merge_upper_y)), RegisterWrite::new(0x100a1, 0), RegisterWrite::new(0x15069, 0), RegisterWrite::new(0x15071, 0), @@ -625,15 +808,15 @@ fn fragment_registers( RegisterWrite::new(0x15109, p.scissor_va), RegisterWrite::new(0x15101, p.depth_bias_va), RegisterWrite::new(0x15021, w.aux_fb_flags), - RegisterWrite::new(0x15211, (p.height as u64) << 32 | p.width as u64), + RegisterWrite::new(0x15211, u64::from(p.height) << 32 | u64::from(p.width)), RegisterWrite::new(0x15049, UNK_15049), RegisterWrite::new(0x10051, w.tib_blocks), RegisterWrite::new(0x15321, p.depth_dimensions), - RegisterWrite::new(0x15301, p.depth_clear as u64), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15301, u64::from(p.depth_clear)), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15311, p.occlusion_query_va), RegisterWrite::new(0x15319, p.zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), RegisterWrite::new(0x15351, 0), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x15331, p.depth.va), @@ -682,11 +865,11 @@ fn fragment_registers( RegisterWrite::new(0x160e0, ids), RegisterWrite::new(0x01499, ids), RegisterWrite::new(0x0a341, ids), - RegisterWrite::new(0x1c838, r.pb_slot as u64), + RegisterWrite::new(0x1c838, u64::from(r.pb_slot)), RegisterWrite::new(0x1ca28, r.pm_scratch), RegisterWrite::new(0x10211, key), RegisterWrite::new(0x10420, key), - RegisterWrite::new(0x14048, r.free_list_slot as u64), + RegisterWrite::new(0x14048, u64::from(r.free_list_slot)), RegisterWrite::new(0x14080, r.fragment_status_va | 1), RegisterWrite::new(0x01731, 1), RegisterWrite::new(0x16020, 1), @@ -716,7 +899,7 @@ fn partial_store_registers( RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -742,13 +925,13 @@ fn partial_resume_registers( RegisterWrite::new(0x15439, p.stencil.compression_stride), RegisterWrite::new(0x15221, 0), RegisterWrite::new(0x15229, 0), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -760,13 +943,13 @@ fn partial_load_registers( RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -902,7 +1085,8 @@ impl TaDescriptor { let geometry = Geometry::new(pass)?; let words = PassWords::new(pass)?; let registers = ta_registers(args, &geometry, &words)?; - let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; + let deflake_2a0_offset = + user_offset32(offset_va(resources.deflake_va, args.deflake.sub[1])?)?; let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; let sampler_max = sampler_max(pass.sampler_count)?; let stamp_value = args.stamp_value(); @@ -919,7 +1103,11 @@ impl TaDescriptor { let scs_register = args.ray_query[0].scratch.enabled(); self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + self.checkpoints = TA_CHECKPOINTS_25G83.map(|count| count + u16::from(scs_register)); + let (registers, count) = ta_registers_25g83(args, &geometry, &words)?; + self.registers.set(array_va, ®isters[..count])?; + } else if scs_register { self.registers.set(array_va, ®isters)?; } else { let mut trimmed = [registers[0]; TA_REGISTERS]; @@ -945,7 +1133,10 @@ impl TaDescriptor { self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; - self.unk_932 = Self::UNK_932; + self.unk_932 = match args.programs == RegisterPrograms::V25G83 { + true => TA_SLOT_INDEX_25G83 + u8::from(scs_register), + false => Self::UNK_932, + }; self.free_list_control_va = resources.free_list_control_va; self.unk_93c = Self::UNK_93C; if args.ordinal == 0 { @@ -969,7 +1160,7 @@ impl TaDescriptor { pass_seq: u8, fragment_busy: bool, ) { - self.fragment_busy = fragment_busy as u32; + self.fragment_busy = u32::from(fragment_busy); self.fragment_kick = fragment.get(); self.pass_seq = [pass_seq; 2]; self.completion = ta.completion_value(pass_seq); @@ -980,11 +1171,14 @@ impl TaDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { - let array = RegisterArrayBinding::new( - offset_va(descriptor_va, Self::ARRAY_OFFSET)?, - TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), - )?; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => TA_CHECKPOINTS_25G83[0] as u8, + false => TA_CHECKPOINTS[0] as u8, + } + u8::from(scs_register); + let array = + RegisterArrayBinding::new(offset_va(descriptor_va, Self::ARRAY_OFFSET)?, checkpoint)?; Ok([Some(array), None, None, None]) } } @@ -1171,13 +1365,18 @@ impl FragmentDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => FRAGMENT_CHECKPOINTS_25G83[0] as usize, + false => FRAGMENT_CHECKPOINTS[0] as usize, + } + usize::from(scs_register); Ok([ - array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, + array(0, checkpoint)?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1249,7 +1448,12 @@ impl FragmentSlot { desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); desc.entries = FRAGMENT_REGISTERS as u16 + extra; let registers = fragment_registers(args, &geometry, &words); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + desc.checkpoints = FRAGMENT_CHECKPOINTS_25G83.map(|count| count + extra); + desc.entries = FRAGMENT_REGISTERS_25G83 as u16 + extra; + let (registers, count) = fragment_registers_25g83(args, &geometry, &words)?; + desc.arrays[0].set(arrays[0], ®isters[..count])?; + } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; @@ -1289,12 +1493,15 @@ impl FragmentSlot { desc.aux_stamp_va = args.fragment.aux_stamp_va; desc.stamp_value = args.stamp_value(); desc.qid = args.fragment.qid.into(); - desc.first_kick = (args.ordinal == 0) as u32; + desc.first_kick = u32::from(args.ordinal == 0); desc.current_id = resources.fragment_ids.current.into(); desc.ordinal = args.ordinal as u32; desc.timestamp_start_va = resources.timestamp_va; desc.timestamp_end_va = timestamp_end_va; - desc.unk_21cc = Self::UNK_21CC; + desc.unk_21cc = match args.programs == RegisterPrograms::V25G83 { + true => FRAGMENT_SLOT_INDEX_25G83 + u8::from(args.ray_query[1].scratch.enabled()), + false => Self::UNK_21CC, + }; desc.free_list_control_va = resources.free_list_control_va; desc.unk_21d6 = Self::UNK_21D6; desc.status_va = resources.fragment_status_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs new file mode 100644 index 0000000000000..2f7b598733368 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware builds. +//! +//! Firmware builds that share the interface version ([`super::initdata::INTERFACE_VERSION`]) +//! still differ in a few shared layouts and queue rules. A [`Variant`] names those differences +//! for one build; each SoC's [`crate::g17::Config`] points at the variant of the firmware it +//! runs. + +/// Where a firmware build keeps the host words of a queue's pointer block +/// ([`super::queue::PointerBlock`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PointerLayout { + /// Offset of the item-ring producer. + pub(crate) producer: usize, + /// Offset of the word the host copies the producer to before it acknowledges a recovery, + /// if the build has one. + pub(crate) recovery_target: Option, +} + +/// How a firmware build acknowledges the initialization root the host hands it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitAck { + /// With an `INITDATA_ACK` message on the firmware endpoint. + Message, + /// By setting word `+0x14` of the primary status A to 1; it sends no message. + Status, +} + +/// The initialization graph of a firmware build. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitGraph { + /// Two instances, as the A18 Pro firmware (26A428) runs. + G17P, + /// One instance carrying both roles, with the 25G83 layouts ([`super::initdata::v25g83`]). + V25G83, +} + +/// The layouts and rules of one G17 firmware build. +pub(crate) struct Variant { + /// The initialization graph. + pub(crate) graph: InitGraph, + /// How the firmware acknowledges its initialization. + pub(crate) init_ack: InitAck, + /// The host ends the power-on transition with `POWER_ON_DONE` once every coprocessor + /// is initialized. + pub(crate) power_on_done: bool, + /// First QID the host may use; the firmware keeps the lower ones for itself. + pub(crate) qid_floor: usize, + /// Host words of a queue's pointer block. + pub(crate) pointers: PointerLayout, + /// The host registers each render pair through the GPU's MMIO queue ports. + pub(crate) mmio_pair_registration: bool, + /// Render kick entries may name an MCache table. + pub(crate) mcache: bool, + /// Renders may set the few-primitives auxiliary framebuffer flag. + pub(crate) few_primitives: bool, + /// The firmware runs ray-traversal work (ray queries and intersection programs). + pub(crate) ray_traversal: bool, + /// Render item rings carry lead commands: InitBufferManager before the first tiling pass + /// of a buffer manager, and FragmentBarrier before every fragment descriptor. + pub(crate) lead_commands: bool, + /// Register programs of the render descriptors. + pub(crate) programs: super::render::RegisterPrograms, +} + +/// A firmware build a SoC supports, identified by the macOS release it ships with. +pub(crate) struct Build { + /// macOS version (major, minor, patch). + pub(crate) os_version: [u32; 3], + /// Name, for the log. + pub(crate) name: &'static str, + /// Layouts and rules of the build. + pub(crate) variant: &'static Variant, +} + +/// How a SoC's firmware build is determined. +pub(crate) enum Firmware { + /// The SoC runs a single build, which is not identified (A18 Pro). + Fixed(&'static Variant), + /// One of these builds, selected by the macOS version that the boot loader reports in + /// `apple,firmware-version`: the release whose boot firmware loaded the GPU firmware, and + /// with it this firmware build. Any other version is refused. + Identified(&'static [Build]), +} + +/// The A18 Pro firmware (26A428). +pub(crate) const V26A428: Variant = Variant { + graph: InitGraph::G17P, + init_ack: InitAck::Message, + power_on_done: true, + qid_floor: 0, + pointers: PointerLayout { + producer: 0x40, + recovery_target: Some(0x50), + }, + mmio_pair_registration: true, + mcache: true, + few_primitives: true, + ray_traversal: true, + lead_commands: false, + programs: super::render::RegisterPrograms::G17P, +}; + +/// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). +pub(crate) const V25G83: Variant = Variant { + graph: InitGraph::V25G83, + init_ack: InitAck::Status, + // Never sent on the M4 Pro: the firmware has retired its opening record when it accepts + // its initialization. + power_on_done: false, + // QIDs below 32 belong to the firmware's bootstrap protocol. + qid_floor: 32, + pointers: PointerLayout { + producer: 0x50, + recovery_target: None, + }, + // Whether this build tolerates the MMIO registration is unknown; the M4 Pro has only run + // without it. + mmio_pair_registration: false, + mcache: false, + // A fragment stage with the few-primitives flag hangs. + few_primitives: false, + // A stalled ray traversal wedges the GPU without a firmware recovery. + ray_traversal: false, + lead_commands: true, + programs: super::render::RegisterPrograms::V25G83, +}; + +const _: () = { + assert!(super::queue::PointerBlock::valid_layout(&V26A428.pointers)); + assert!(super::queue::PointerBlock::valid_layout(&V25G83.pointers)); +}; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index d0dcc85c7f09e..10af261aeb3ac 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -16,6 +16,8 @@ use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; +mod v25g83; + const EVENT_RING_RECHECK: u32 = 256; const INDEX_SIZE: usize = 0x10000; @@ -121,7 +123,7 @@ impl WorkPublication<'_> { // fully initialized padding-free ABI value, and the device mutex owns publication. unsafe { core::ptr::copy_nonoverlapping( - self.record as *const _ as *const u8, + core::ptr::from_ref(self.record).cast::(), self.destination, core::mem::size_of_val(self.record), ); @@ -155,6 +157,15 @@ pub(super) struct InitData { boot_control: usize, boot_state: usize, metrics_fw_va: u64, + /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). + pm_metrics_va: u64, + /// `Config::recovery_slot_keys`. + recovery_slot_keys: Option, + /// Primary status A and status B in the cluster. + status_a: usize, + status_b: usize, + /// Separately allocated views; `None` when they are packed into the bundle. + views: Option<[usize; 5]>, compute_shared: Option, compute_scheduler: Option, compute_active: core::cell::Cell, @@ -174,6 +185,8 @@ struct BuildEnv<'a> { render: &'a mmu::Vm, base: u64, dynamic: Range, + /// GPU clusters, as the GPU reports them. + clusters: u32, } impl BuildEnv<'_> { @@ -235,6 +248,11 @@ impl InitData { boot_control: 0, boot_state: 0, metrics_fw_va: 0, + pm_metrics_va: 0, + recovery_slot_keys: None, + status_a: abi::private::PRIMARY_STATUS_A, + status_b: abi::private::STATUS_B, + views: None, compute_shared: None, compute_scheduler: None, compute_active: core::cell::Cell::new(false), @@ -247,6 +265,10 @@ impl InitData { } } + pub(super) fn pm_metrics_va(&self) -> u64 { + self.pm_metrics_va + } + pub(super) fn metrics_fw_va(&self) -> Result { if self.metrics_fw_va == 0 { return Err(EIO); @@ -363,7 +385,7 @@ impl InitData { .initialize::<[u64; low::RENDER_RUN_LIST_SIZE / 8]>(0, |runs| { for (index, run) in runs.iter_mut().take(COMPUTE_BLOCKS_POPULATED).enumerate() { *run = (low::compute_block(index) & COMPUTE_RUN_ADDRESS_MASK) - | ((abi::FREE_LIST_BLOCK_PAGES as u64) << 52); + | (u64::from(abi::FREE_LIST_BLOCK_PAGES) << 52); } Ok(()) })?; @@ -502,7 +524,7 @@ impl InitData { core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -522,7 +544,7 @@ impl InitData { ) -> Result<[u32; 3]> { use super::fw::channels; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; for (value, offset) in values.iter_mut().zip([ @@ -601,7 +623,7 @@ impl InitData { }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -694,7 +716,12 @@ impl InitData { uat: &mmu::Uat, render: &mmu::Vm, metrics: &KernelObject, + clusters: u32, + active_cores: u32, + variant: &super::fw::Variant, ) -> Result { + self.pm_metrics_va = cfg.pm_metrics.va; + self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { dev, @@ -703,9 +730,13 @@ impl InitData { render, base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, + clusters, }; - self.map_register_windows(&env)?; - let tables = self.build_descriptor_tables(&env)?; + if variant.graph == super::fw::variant::InitGraph::V25G83 { + return self.build_25g83(&env, cfg, metrics, active_cores); + } + self.map_register_windows(&env, cfg.windows)?; + let tables = self.build_descriptor_tables(&env, abi::private::SIZE)?; self.build_render_free_lists(&env)?; let first_block = self.build_render_pool(&env)?; self.build_compute_pools(&env, first_block)?; @@ -713,31 +744,41 @@ impl InitData { self.initialize_firmware_objects(&env, cfg, &tables, &bundle) } - fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { - for window in &t8140::REGISTER_WINDOWS { - let (phys, va, size) = window.mapping().ok_or(EINVAL)?; - self.aliases.push( - env.vm.map_io( - env.dev.as_ref().as_ref(), - va, - phys.try_into()?, - size.try_into()?, - mmu::PROT_FW_MMIO_RW, - )?, - GFP_KERNEL, - )?; + fn map_register_windows( + &mut self, + env: &BuildEnv<'_>, + windows: &[abi::RegisterWindow], + ) -> Result { + for window in windows { + for instance in 0..window.instances { + let (phys, va, size) = window.mapping(instance).ok_or(EINVAL)?; + self.aliases.push( + env.vm.map_io( + env.dev.as_ref().as_ref(), + va, + phys.try_into()?, + size.try_into()?, + mmu::PROT_FW_MMIO_RW, + )?, + GFP_KERNEL, + )?; + } } Ok(()) } /// The private cluster, zero page and the three descriptor tables with their low aliases. - fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { + fn build_descriptor_tables( + &mut self, + env: &BuildEnv<'_>, + cluster_size: usize, + ) -> Result { use mmu::{ PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; - self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? @@ -939,12 +980,12 @@ impl InitData { low_va: 0, high_va: 0, }; 2]; - for slot in 0..2 { + for (slot, completion_ring) in completion_rings.iter_mut().enumerate() { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; self.completion_rings[slot] = ring; self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; - completion_rings[slot] = abi::CompletionRing { + *completion_ring = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], high_va: env.base + offset, }; @@ -988,13 +1029,13 @@ impl InitData { bundle: &BundleObjects, ) -> Result { let base = env.base; - let opps = Self::opps(env.dev)?; + let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, - windows: &t8140::REGISTER_WINDOWS, - reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, opps: &opps, - perf: &t8140::PERF, + perf: cfg.perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1002,7 +1043,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(); + region.init(cfg.region_c); Ok(()) })?; self.object_mut(self.cluster)? @@ -1050,8 +1091,8 @@ impl InitData { let slot = role as usize; let main_offset = abi::bundle::MAIN_CONFIG[slot]; let main_va = base + high::BUNDLE + main_offset as u64; - let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; - let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; + let state_va = base + high::PRIVATE_CLUSTER + self.state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + self.status_offset(role) as u64; let args = abi::MainConfigArgs { bundle_va: base + high::BUNDLE, main_config_va: main_va, @@ -1107,7 +1148,7 @@ impl InitData { (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), ] { self.object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .store(value, Ordering::Relaxed); } } @@ -1150,7 +1191,7 @@ impl InitData { )?; } mappings.push( - metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, + metrics.map_alias(vm, self.pm_metrics_va, mmu::PROT_GPU_FW_SHARED_RW)?, GFP_KERNEL, )?; Ok(mappings) @@ -1180,37 +1221,65 @@ impl InitData { .load(Ordering::Relaxed)) } - fn opps(dev: &AsahiDrmDevice) -> Result> { + /// The GPU's operating points: `count` of them, each with one voltage cell for the whole + /// GPU or one per cluster. + fn opps(dev: &AsahiDrmDevice, count: usize, clusters: u32) -> Result> { + let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; + if opps.len() != count { + return Err(EINVAL); + } + let cells = opps.first().map_or(0, |opp| opp.cells); + if cells != 1 && u32::try_from(cells) != Ok(clusters) { + dev_err!( + dev.as_ref(), + "GPU operating points have {} voltage cells, expected 1 or {}\n", + cells, + clusters + ); + return Err(EINVAL); + } + Ok(opps) + } + + /// The operating points of the table that the GPU node's property `name` names; all have + /// the same number of voltage cells. + fn opp_table(dev: &AsahiDrmDevice, name: &kernel::str::CStr) -> Result> { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let table = node - .parse_phandle(c_str!("operating-points-v2"), 0) - .ok_or(EIO)?; + let table = node.parse_phandle(name, 0).ok_or(EIO)?; let mut opps = KVec::new(); for opp in table.children() { + // One cell for the whole GPU, or one per cluster. + let cells: KVec = opp.get_property(c_str!("opp-microvolt"))?; + if cells.is_empty() || cells.len() > abi::OPP_CELLS { + return Err(EINVAL); + } + let mut volt_uv = [0; abi::OPP_CELLS]; + volt_uv[..cells.len()].copy_from_slice(&cells); opps.push( abi::Opp { freq_hz: opp.get_property(c_str!("opp-hz"))?, - volt_uv: opp.get_property(c_str!("opp-microvolt"))?, + volt_uv, + cells: cells.len(), }, GFP_KERNEL, )?; } - if opps.len() != t8140::OPPS { + if opps.iter().any(|opp| opp.cells != opps[0].cells) { return Err(EINVAL); } Ok(opps) } - fn state_offset(role: Role) -> usize { + fn state_offset(&self, role: Role) -> usize { match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, } } - fn status_offset(role: Role) -> usize { + fn status_offset(&self, role: Role) -> usize { match role { - Role::Primary => abi::private::PRIMARY_STATUS_A, + Role::Primary => self.status_a, Role::Secondary => abi::private::SECONDARY_STATUS_A, } } @@ -1218,7 +1287,7 @@ impl InitData { /// Recheck the host-authored event pointer pair after a bounded number of drains. /// The device mutex serializes the cache and all event consumption. pub(super) fn validate_event_ring(&self, role: Role) -> Result { - let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; + let status = self.object(self.cluster)?.gpu_va() + self.status_offset(role) as u64; let expected = ( status + abi::status::EVENT_STATE, status + abi::status::EVENT_RING, @@ -1247,9 +1316,14 @@ impl InitData { Ok(()) } + /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). + pub(super) fn init_accepted(&self) -> Result { + Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + } + pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { self.object(self.cluster)?.word( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1261,7 +1335,7 @@ impl InitData { offset: usize, ) -> Result<[u32; N]> { self.object(self.cluster)?.read_words( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1336,7 +1410,7 @@ impl InitData { for offset in [0, 0x10, abi::ChannelState::PRODUCER] { if self .object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::SeqCst) != abi::OPENING_PRODUCER { @@ -1351,8 +1425,12 @@ impl InitData { /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; - let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + // Separately allocated views live at offset 0 of their own object. + let (object, base) = match self.views { + Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), + None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), + }; let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { @@ -1363,20 +1441,20 @@ impl InitData { fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; - if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + if actual != object.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { return Err(EINVAL); } - bundle.read_words(base + offset) + object.read_words(base + offset) } } impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { let mask = self.object(self.cluster)? - .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot)?; + let key = super::recovery::slot_key(slot, self.recovery_slot_keys)?; let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) .ok().map(|words| words[0])?; @@ -1391,21 +1469,21 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1417,17 +1495,21 @@ impl super::recovery::Memory for InitData { } fn clear_timestamps(&self) -> Result { - let bundle = self.object(self.bundle)?; + // The per-QID timestamp table is view 4; separately allocated views live at offset 0 of + // their own object. + let (table, offset) = match self.views { + Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), + None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), + }; // Validate the complete table before publishing its first live word. const ENTRIES: usize = 128; const STRIDE: usize = 0x10; - let offset = abi::bundle::VIEWS[4]; - bundle.pointer(offset, ENTRIES * STRIDE)?; + table.pointer(offset, ENTRIES * STRIDE)?; for qid in 0..ENTRIES { - bundle + table .word(offset + qid * STRIDE)? .store(0, Ordering::Relaxed); - bundle + table .dword(offset + qid * STRIDE + 8)? .store(0, Ordering::Relaxed); } @@ -1437,7 +1519,7 @@ impl super::recovery::Memory for InitData { fn entries(&self) -> Result { let cluster = self.object(self.cluster)?; - let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; + let base = self.status_b + abi::PrimaryStatusB::RECOVERY_INFO; fence(Ordering::Acquire); let mut entries = super::recovery::Entries::default(); for index in 0..256 { diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs new file mode 100644 index 0000000000000..85aa6468fa311 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The initialization graph of the 25G83 firmware build. +//! +//! One coprocessor carries both roles: there is one main configuration, with the +//! secondary-only words, and one merged root. The cluster, the hardware data, the power +//! configuration, region C, the views and the firmware area follow the 25G83 layouts +//! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. + +use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; +use crate::g17::{Config, Role}; +use crate::mmu; +use abi::v25g83 as v; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, prelude::*}; + +/// Object indices of the 25G83-specific objects. +struct Objects { + views: [usize; 5], + firmware_area: usize, + power_state: usize, + aon: usize, +} + +impl InitData { + /// Constructs the 25G83 graph after the firmware has published its root entries. + pub(super) fn build_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + metrics: &KernelObject, + active_cores: u32, + ) -> Result { + self.status_a = v::private::STATUS_A; + self.status_b = v::private::STATUS_B; + self.map_register_windows(env, cfg.windows)?; + let tables = self.build_descriptor_tables(env, v::private::SIZE)?; + self.build_render_free_lists(env)?; + let first_block = self.build_render_pool(env)?; + self.build_compute_pools(env, first_block)?; + let bundle = self.build_bundle(env, cfg, metrics)?; + let objects = self.build_25g83_objects(env)?; + self.views = Some(objects.views); + self.initialize_25g83(env, cfg, active_cores, &tables, &bundle, &objects)?; + if cfg.uat_quirks.clean_tables { + // Write back every host-built byte before the firmware may read it. + for object in self.objects.iter_mut() { + let bytes = object.bytes_mut(); + crate::pgtable::clean_poc(bytes.as_ptr(), bytes.len()); + } + crate::pgtable::poc_barrier(); + } + Ok(()) + } + + /// The views, the firmware area, the power-state object and the AON table. + fn build_25g83_objects(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS}; + use CpuMap::WriteCombined as WC; + let mut views = [0; 5]; + for (view, offset) in views.iter_mut().zip(v::kernel_window::VIEWS) { + *view = self.add(env.fixed(offset, mmu::UAT_PGSZ, FP, WC)?)?; + } + let firmware_area = + self.add(env.fixed(v::kernel_window::FIRMWARE_AREA, mmu::UAT_PGSZ, FP, WC)?)?; + let power_state = + self.add(env.fixed(v::kernel_window::POWER_STATE, mmu::UAT_PGSZ, FS, WC)?)?; + let aon = self.add(env.fixed(v::kernel_window::AON_TABLE, mmu::UAT_PGSZ, FP, WC)?)?; + Ok(Objects { + views, + firmware_area, + power_state, + aon, + }) + } + + /// Writes the 25G83 firmware objects, checks them against the expected contents, then + /// stages the opening control-ring state. + fn initialize_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + active_cores: u32, + tables: &DescriptorTables, + bundle: &BundleObjects, + objects: &Objects, + ) -> Result { + let bundle_va = self.object(self.bundle)?.gpu_va(); + let cluster_va = self.object(self.cluster)?.gpu_va(); + let config_va = cluster_va + v::private::CONFIG as u64; + let status_a_va = cluster_va + v::private::STATUS_A as u64; + let status_b_va = cluster_va + v::private::STATUS_B as u64; + let aon_va = self.object(objects.aon)?.gpu_va(); + let power_state_va = self.object(objects.power_state)?.gpu_va(); + let firmware_area_va = self.object(objects.firmware_area)?.gpu_va(); + let mut view_va = [0; 5]; + for (va, index) in view_va.iter_mut().zip(objects.views) { + *va = self.object(index)?.gpu_va(); + } + + let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; + let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; + let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; + let hw_args = abi::HwDataArgs { + chip_id: cfg.chip_id, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, + opps: &opps, + perf: cfg.perf, + completion_rings: bundle.completion_rings, + qos_va: self.object(bundle.qos)?.gpu_va(), + }; + let hw_args_25g83 = v::HwDataArgs25G83 { + active_cores, + cs_opps: &cs_opps, + afr_opps: &afr_opps, + }; + // SAFETY: Plain C entropy source. + let entropy = unsafe { kernel::bindings::get_random_u32() }; + let mut hashes = [0; 3]; + self.object_mut(self.bundle)? + .initialize::(0, |hw| { + hw.init(&hw_args, &hw_args_25g83)?; + hashes[0] = hw.hash(); + Ok(()) + })?; + self.object_mut(self.bundle)? + .initialize::(v::POWER_CONFIG, |power| { + power.init(config_va, aon_va, entropy); + hashes[1] = power.hash(); + Ok(()) + })?; + self.object_mut(self.region_c)? + .initialize::(0, |region| { + region.init(cfg.region_c); + Ok(()) + })?; + hashes[2] = v::hash_without(self.object_mut(self.region_c)?.bytes_mut(), &[]); + // The constant contents must equal what the Gravity Linux encoders produce on the proven + // setup; the operating points and the core count come from the device and are not hashed. + let expected = [ + v::expected::HW_DATA, + v::expected::POWER_CONFIG, + v::expected::REGION_C, + ]; + if hashes != expected { + dev_err!( + env.dev.as_ref(), + "25G83 initialization data differs: {:#x?}, expected {:#x?}\n", + hashes, + expected + ); + return Err(EINVAL); + } + for (index, word) in v::AON_TABLE.iter().enumerate() { + self.object_mut(objects.aon)?.write(index * 4, *word)?; + } + + // Status A and B: only the boot words and the firmware-control pointers. + self.object_mut(self.cluster)? + .write(v::private::STATUS_A, abi::StatusBlock::new())?; + self.object_mut(self.cluster)? + .write(v::private::STATUS_B, abi::StatusBlock::new())?; + // Status B keeps the A18 Pro's field offsets up to its recovery words. + let fwctl_va = self.object(tables.fwctl)?.gpu_va(); + let fwctl = + v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); + let cluster = self.object(self.cluster)?; + cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); + cluster.dword(fwctl + 8)?.store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); + + // One main configuration carrying both roles. + let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; + let main_va = bundle_va + main_offset as u64; + let args = abi::MainConfigArgs { + bundle_va, + main_config_va: main_va, + state_grid_va: cluster_va + self.state_offset(Role::Primary) as u64, + status_a_va, + pb_descriptors_low_va: low::PB_DESCRIPTORS, + page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, + }; + let views = abi::PrimaryViews { + zero_page_va: self.object(bundle.zero)?.gpu_va(), + pb_descriptors_va: self.object(tables.pb)?.gpu_va(), + page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), + }; + self.object_mut(self.bundle)? + .initialize::(main_offset, |main| { + main.init_primary(&args, &views); + main.view_va = view_va; + main.firmware_area_va = [firmware_area_va; 2]; + main.unk_18 = (cluster_va + v::private::MAIN_HEADER as u64).to_le_bytes(); + // The secondary role's words, on the single instance. + main.unk_300 = 4; + main.power_config_va = bundle_va + v::POWER_CONFIG as u64; + main.unk_32c = 0; + main.unk_344 = 0; + Ok(()) + })?; + self.object_mut(self.bundle)?.write( + main_offset + abi::MainConfig::CONTROL_RING, + abi::OpeningRecord::new(abi::Role::Primary), + )?; + + // The merged root. + let root = abi::RootArgs { + region_a_va: self.object(bundle.region_a)?.gpu_va(), + region_c_va: self.object(self.region_c)?.gpu_va(), + main_config_va: main_va, + status_a_va, + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + self.object_mut(self.roots)?.write( + 0, + abi::SecondaryRoot::merged(&root, status_b_va, power_state_va, config_va), + )?; + for &(offset, value, repeat) in v::POWER_STATE_WORDS { + for index in 0..repeat { + self.object_mut(objects.power_state)?.write(offset + 4 * index, value)?; + } + } + + // The opening control record is staged in slot 0. + for (offset, value) in [ + (0, 0), + (0x10, 0), + (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), + ] { + self.object(self.cluster)? + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? + .store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b5a5b3759de6d..61b8c3b7b4737 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -13,7 +13,7 @@ use kernel::{ sync::Arc, }; -pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; +pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; const RENDER_CREDITS: u32 = 32; const COMPUTE_CREDITS: u32 = 16; @@ -143,23 +143,40 @@ impl Packet { } } + /// Replay holds the device mutex. Its original inputs were satisfied before + /// first publication; never sleep on a fence that needs event service here. + pub(crate) fn check_replay_dependencies(&self) -> Result { + for fence in &self.inputs { + super::dependency::replay_status(fence_status(fence))?; + } + for fence in &self.firmware { check_status(fence)?; } + Ok(()) + } + /// Scheduler dependency readiness includes failed fences. Consume the actual /// status before publication, and retain the references through that check. pub(crate) fn check_dependencies(&self) -> Result { + let polls = if self.command.long_job() { 5 } else { 1 }; + let deadline = super::now_ns().saturating_add(u64::from(TIMEOUT_MS) * polls * 1_000_000); for fence in &self.inputs { - // SAFETY: The retained fence owns the pointer throughout the wait and read. - let waited = unsafe { - bindings::dma_fence_wait_timeout( - fence.raw(), - true, - kernel::task::MAX_SCHEDULE_TIMEOUT, - ) - }; - if waited < 0 { - return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); - } - if waited == 0 { - return Err(ECANCELED); + loop { + if self.cancel_requested() || self.completion.status().get() != 0 { + return Err(ECANCELED); + } + if fence_status(fence) != 0 { break; } + if super::now_ns() >= deadline { return Err(ECANCELED); } + // SAFETY: The retained fence owns the pointer throughout the bounded wait. + let waited = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + true, + kernel::time::msecs_to_jiffies(10) as isize, + ) + }; + if waited < 0 { + return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + } + if waited > 0 { break; } } check_status(fence)?; } @@ -179,6 +196,13 @@ impl Packet { .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) .is_ok() } + /// Gives up a claim taken from the submitting ioctl before the job was + /// pushed: nothing became visible, and the scheduler claims it again. + pub(crate) fn abandon_publication(&self) { + let _ = self + .publication + .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Acquire); + } pub(crate) fn mark_published(&self) { self.publication.store(2, Ordering::Release); } @@ -201,7 +225,7 @@ impl Packet { } pub(crate) fn set_early_result(&self, result: Result) { self.early.store( - result.map_or_else(|error| error.to_errno(), |_| 1), + result.map_or_else(|error| error.to_errno(), |()| 1), Ordering::Release, ); } @@ -242,7 +266,8 @@ pub(crate) trait Backend: Send + Sync { None } - /// Lazily create or rebind this logical queue's physical compute graph. + /// Validate compute sizing without reserving a physical channel. Queued + /// commands may wait on a host signal submitted after this ioctl returns. fn ensure_compute(&self, scs_pages: u16) -> Result; fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { if pages != [0; 2] { return Err(EINVAL); } @@ -271,6 +296,7 @@ struct Job { struct Timeout { backend: Arc, victim_renewals: u8, + execution_renewals: u8, packet: Arc, } @@ -287,12 +313,14 @@ impl Drop for Job { impl sched::JobImpl for Job { type TimeoutData = Timeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + const MODULE: Option<&'static kernel::ThisModule> = + Some(&crate::THIS_MODULE); fn timeout_data(&self) -> Timeout { Timeout { backend: self.backend.clone(), victim_renewals: 0, + execution_renewals: 0, packet: self.packet.clone(), } } @@ -301,9 +329,14 @@ impl sched::JobImpl for Job { if finished || job.packet.completion.take_replay_timeout() { return true; } + if super::timeout::renew_execution( + job.packet.is_published(), job.packet.command.long_job(), &mut job.execution_renewals, + ) { + return true; + } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); - if deferred && job.packet.engine() == Engine::Compute { - job.victim_renewals += 1; + if deferred { + job.victim_renewals = job.victim_renewals.saturating_add(1); } deferred } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 3b78b9443a86d..c337a8b298f19 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,8 +6,15 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; +use core::{ + mem::ManuallyDrop, + sync::atomic::{fence, Ordering}, +}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; use super::{ fw::{kick::*, queue::DataMaster}, @@ -17,8 +24,24 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// QIDs kept free for render pairs when compute queues fill the table. -const COMPUTE_RENDER_RESERVE: usize = 16; +/// Minimum render-channel capacity protected from compute-only allocation. +// Protect one tiling/fragment pair for each logical execution context, plus +// the two unpublished bootstrap reservations. The remaining ten compute +// channels are shared after exact retirement, rather than pinned per client. +const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; +pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; +/// Logical queues that can each hold a render pair: what the render reserve leaves beside the +/// two bootstrap reservations once the QIDs below `floor`, which the firmware keeps, are gone +/// (every execution context with no floor). +pub(super) const fn queue_limit(floor: usize) -> u32 { + ((COMPUTE_RENDER_RESERVE - 2 - floor) / 2) as u32 +} + +const _: () = { + assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); + assert!(COMPUTE_CHANNELS >= 4); + assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); +}; /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -47,19 +70,42 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], - compute_publication: u64, + initial_render: [Option; 2], + publication: u64, leased: usize, + /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). + floor: usize, } impl QueueIds { - pub(crate) fn new() -> impl Init { + pub(crate) fn new(floor: usize) -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), - compute_publication: 0, + initial_render: [None; 2], + publication: 0, leased: 0, + // The reserve must keep render capacity above the floor. + floor: if floor < COMPUTE_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } + /// Keep the initial render reservations until another engine needs their + /// capacity. No graph receives these tokens, so an unpublished reservation + /// can be replaced without changing an installed physical channel. + pub(crate) fn reserve_initial_render(&mut self) -> Result { + if self.leased != 0 { + return Err(EINVAL); + } + for (qid, engine) in [DataMaster::Tiling, DataMaster::Fragment] + .into_iter() + .enumerate() + { + let id = self.reserve(qid as u64 + 1, engine, Some(qid as u8))?; + self.initial_render[qid] = Some(id); + } + Ok(()) + } + /// Reuses only an exact owner/engine match. Preferred placement is for /// bootstrapping; it does not reserve a partition for an engine. pub(crate) fn reserve( @@ -83,16 +129,56 @@ impl QueueIds { Err(EINVAL) }; } - // Healthy existing channels keep their QID. Never recycle a published - // identity to recover capacity after a fault; leave eight render pairs. - if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(ENOSPC); + // The QIDs below the floor are never the host's; the reserve shrinks + // with them, so the compute channel count stays the same. + let total = QID_COUNT - self.floor; + let reserve = COMPUTE_RENDER_RESERVE - self.floor; + let initial = if engine == DataMaster::Compute && self.leased == total { + self.initial_render + .iter() + .copied() + .enumerate() + .find_map(|(slot, id)| { + id.filter(|id| self.can_cancel_unpublished(*id)) + .map(|id| (slot, id)) + }) + } else { + None + }; + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Published identities always retain their original lifetime rules. + if engine == DataMaster::Compute { + let render = self + .entries + .iter() + .flatten() + .filter(|entry| { + matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) + }) + .count(); + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let replacement = usize::from(initial.is_some()); + if total - self.leased + replacement + <= reserve.saturating_sub(render - replacement) + { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); + } } let index = preferred - .map(usize::from) + .map(|index| usize::from(index) + self.floor) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or(ENOSPC)?; + .or_else(|| { + self.entries + .iter() + .skip(self.floor) + .position(Option::is_none) + .map(|index| index + self.floor) + }) + .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) + .ok_or_else(|| crate::g17::capacity::refused(crate::g17::capacity::Site::PhysicalQids))?; let id = Id { qid: index as u8, owner, @@ -102,14 +188,19 @@ impl QueueIds { id, published: false, }); - self.leased += 1; + if let Some((slot, _)) = initial { + self.initial_render[slot] = None; + } else { + self.leased += 1; + } Ok(id) } - /// Ordered under the device mutex; no atomic operation on the submission path. - pub(crate) fn next_compute_publication(&mut self) -> Result { - self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; - Ok(self.compute_publication) + /// Ordered under the device mutex; no atomic operation on the submission path. Compute and + /// render publications share it, so a watchdog can order work across both engines. + pub(crate) fn next_publication(&mut self) -> Result { + self.publication = self.publication.checked_add(1).ok_or(EOVERFLOW)?; + Ok(self.publication) } /// Call after preflighting the complete installation, before any address @@ -127,6 +218,13 @@ impl QueueIds { Ok(()) } + pub(crate) fn can_cancel_unpublished(&self, id: Id) -> bool { + self.entries + .get(usize::from(id.qid)) + .and_then(Option::as_ref) + .is_some_and(|entry| entry.id == id && !entry.published) + } + pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; match slot { @@ -140,6 +238,82 @@ impl QueueIds { } } +#[kunit_tests(asahi_g17_qid_accounting)] +mod accounting_tests { + use super::*; + + fn table() -> KBox { + KBox::init(QueueIds::new(0), GFP_KERNEL).unwrap() + } + + #[test] + fn render_identities_satisfy_the_compute_reserve() { + let mut ids = table(); + for owner in 1..=COMPUTE_RENDER_RESERVE as u64 { + ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + } + for owner in COMPUTE_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { + ids.reserve(owner, DataMaster::Compute, None).unwrap(); + } + assert_eq!(ids.leased, QID_COUNT); + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + assert_eq!(ids.leased, QID_COUNT); + } + + #[test] + fn compute_cannot_consume_the_render_minimum() { + let mut ids = table(); + for owner in 1..=(QID_COUNT - COMPUTE_RENDER_RESERVE) as u64 { + ids.reserve(owner, DataMaster::Compute, None).unwrap(); + } + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); + assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); + ids.cancel_unpublished(tiling).unwrap(); + assert_eq!(ids.leased, QID_COUNT - COMPUTE_RENDER_RESERVE); + } + + #[test] + fn initial_reservations_yield_but_published_ids_do_not() { + let mut ids = table(); + ids.reserve_initial_render().unwrap(); + for owner in 3..=QID_COUNT as u64 { + ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + } + let first = ids.reserve(1000, DataMaster::Compute, None).unwrap(); + let second = ids.reserve(1001, DataMaster::Compute, None).unwrap(); + assert_ne!(first.qid(), second.qid()); + assert!(ids.initial_render.iter().all(Option::is_none)); + assert_eq!(ids.leased, QID_COUNT); + ids.publish(first).unwrap(); + assert_eq!(ids.cancel_unpublished(first), Err(EINVAL)); + assert_eq!(ids.leased, QID_COUNT); + ids.cancel_unpublished(second).unwrap(); + assert_eq!(ids.leased, QID_COUNT - 1); + assert_eq!(ids.cancel_unpublished(second), Err(EINVAL)); + assert_eq!(ids.leased, QID_COUNT - 1); + } + + #[test] + fn failed_second_render_id_returns_only_the_unpublished_first() { + let mut ids = table(); + for owner in 1..QID_COUNT as u64 { + let id = ids.reserve(owner, DataMaster::Tiling, None).unwrap(); + ids.publish(id).unwrap(); + } + let first = ids.reserve(1000, DataMaster::Tiling, None).unwrap(); + assert_eq!(ids.reserve(1001, DataMaster::Fragment, None), Err(ENOSPC)); + ids.cancel_unpublished(first).unwrap(); + assert_eq!(ids.leased, QID_COUNT - 1); + let replacement = ids.reserve(1002, DataMaster::Compute, None).unwrap(); + assert_eq!(replacement.qid(), first.qid()); + assert_ne!(replacement, first); + assert_eq!(ids.reserve(1002, DataMaster::Compute, None), Ok(replacement)); + assert_eq!(ids.leased, QID_COUNT); + assert_eq!(ids.reserve(1002, DataMaster::Fragment, None), Err(EINVAL)); + } +} + // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { @@ -147,10 +321,94 @@ enum Backing { high: mmu::KernelMapping, low: KernelObject, }, - Render { - low: mmu::KernelMapping, - high: KernelObject, - }, + Render(RenderRing), +} + +/// A render ring's pages and the shared lower reservation clients alias. +struct RenderStorage { + // The lower alias drops before its backing. + low: mmu::KernelMapping, + high: KernelObject, +} + +/// Render ring storage outlives its rings for the life of the device. +/// +/// A client VM keeps each render ring alias it installs until the VM closes, +/// while an installed pair whose identities were never published can be +/// reclaimed and its rings dropped. Freeing that storage would hand its shared +/// lower address to another ring; a client still aliasing the old pages there +/// that later receives the new ring could not alias it (`prepare_client_alias` +/// fails with EFAULT). Retired storage is therefore kept and reused by later +/// rings: an address keeps naming the same pages, so every alias stays valid. +#[pin_data] +pub(crate) struct RenderRings { + #[pin] + stock: Mutex, +} + +struct RingStock { + /// Retired storage; `true` marks storage of a ring that never wrote an + /// entry, which a later ring may reuse. Written rings stay retired. + retired: KVec<(bool, RenderStorage)>, + /// Storage created so far; `retired` keeps capacity for all of it, so a + /// dropped ring always returns its storage without allocating. + created: usize, +} + +impl RenderRings { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + stock <- new_mutex!( + RingStock { retired: KVec::new(), created: 0 }, + "G17 render kick rings" + ), + }), + GFP_KERNEL, + ) + } + + fn take(&self) -> Option { + let mut stock = self.stock.lock(); + let index = stock.retired.iter().rposition(|(reusable, _)| *reusable)?; + let last = stock.retired.len() - 1; + stock.retired.swap(index, last); + stock.retired.pop().map(|(_, storage)| storage) + } + + /// Reserves the return of one more, just created storage. + fn admit(&self) -> Result { + let mut stock = self.stock.lock(); + let created = stock.created.checked_add(1).ok_or(EOVERFLOW)?; + let additional = created.saturating_sub(stock.retired.len()); + stock.retired.reserve(additional, GFP_KERNEL)?; + stock.created = created; + Ok(()) + } + + fn put(&self, storage: RenderStorage, reusable: bool) { + let mut stock = self.stock.lock(); + if let Err(rejected) = stock.retired.push_within_capacity((reusable, storage)) { + // `admit` reserved room for every created storage. Never free it. + core::mem::forget(rejected); + } + } +} + +/// Storage lent to one render ring; returned to the stock when the ring drops. +struct RenderRing { + storage: ManuallyDrop, + rings: Arc, + /// Set once an entry was written: such storage is never reused. + written: bool, +} + +impl Drop for RenderRing { + fn drop(&mut self) { + // SAFETY: `storage` is taken exactly once, here, and not used afterwards. + let storage = unsafe { ManuallyDrop::take(&mut self.storage) }; + self.rings.put(storage, !self.written); + } } /// An installed queue's permanent ring. Client aliases are held by the bound @@ -186,6 +444,9 @@ impl PreparedKick<'_> { // the new entry starts before the outer command is published. unsafe { self.destination.write_unaligned(self.entry) }; fence(Ordering::SeqCst); + if let Backing::Render(ring) = &mut self.queue.backing { + ring.written = true; + } self.queue.previous = Some(self.timestamp); self.queue.timestamp = self.timestamp.next(); self.timestamp @@ -220,14 +481,45 @@ impl Queue { }) } + /// Retired storage of a never-written ring is reused first. Its address + /// names the same pages in every client that aliases it, so an existing + /// alias in `client` is kept (None) and a missing one is created. pub(crate) fn render( alloc: &Allocator<'_>, id: Id, client: &mmu::Vm, - ) -> Result<(Self, mmu::KernelMapping)> { + rings: &Arc, + ) -> Result<(Self, Option)> { if id.engine == DataMaster::Compute { return Err(EINVAL); } + if let Some(storage) = rings.take() { + let mut ring = RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }; + // A fresh ring is zeroed; no firmware ever read this one. + ring.storage.high.bytes_mut().fill(0); + fence(Ordering::SeqCst); + let address = ring.storage.low.iova(); + let alias = + ring.storage + .high + .prepare_alias(client, address, mmu::PROT_GPU_SHARED_RW, true); + // On failure the storage returns to the stock and a fresh ring is built. + if let Ok(alias) = alias { + return Ok(( + Self { + backing: Backing::Render(ring), + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + alias, + )); + } + } let high = alloc.kernel( KICK_RING_SIZE, mmu::UAT_PGSZ as u64, @@ -255,14 +547,26 @@ impl Queue { Err(error) => return Err(error), } }; + let storage = RenderStorage { low, high }; + // Count the storage only once it exists. Without room to return it, + // nothing has aliased it but `client`, which drops first. + if let Err(error) = rings.admit() { + drop(client); + drop(storage); + return Err(error); + } Ok(( Self { - backing: Backing::Render { low, high }, + backing: Backing::Render(RenderRing { + storage: ManuallyDrop::new(storage), + rings: rings.clone(), + written: false, + }), id, timestamp: KickTimestamp::FIRST, previous: None, }, - client, + Some(client), )) } @@ -272,19 +576,19 @@ impl Queue { pub(crate) fn low_va(&self) -> u64 { match &self.backing { Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render { low, .. } => low.iova(), + Backing::Render(ring) => ring.storage.low.iova(), } } pub(crate) fn firmware_va(&self) -> u64 { match &self.backing { Backing::Compute { high, .. } => high.iova(), - Backing::Render { high, .. } => high.gpu_va(), + Backing::Render(ring) => ring.storage.high.gpu_va(), } } fn object(&mut self) -> &mut KernelObject { match &mut self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &mut ring.storage.high, } } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { @@ -297,7 +601,7 @@ impl Queue { pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { let object = match &self.backing { Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, + Backing::Render(ring) => &ring.storage.high, }; object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) } @@ -353,3 +657,20 @@ impl Queue { }) } } + +impl QueueIds { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let mut engines = [0usize; 3]; + let mut published = 0; + for entry in self.entries.iter().flatten() { + engines[match entry.id.engine { + DataMaster::Tiling => 0, + DataMaster::Fragment => 1, + DataMaster::Compute => 2, + }] += 1; + published += usize::from(entry.published); + } + writeln!(f, "physical_qids used={} total={} published={} tiling={} fragment={} compute={}", + self.leased, QID_COUNT, published, engines[0], engines[1], engines[2]) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 9da90d2b0843a..2999daec47015 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -23,6 +23,7 @@ mod command; pub(crate) use command::Attachments; mod completion; pub(crate) mod context; +mod data_segment; mod dependency; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; @@ -35,13 +36,15 @@ mod ids; pub(crate) mod hazard; mod initdata; mod job; +mod capacity; mod kick; mod object; -mod power; +pub(crate) mod power; mod preparation; mod qos; mod queue; mod recovery; +mod replay; mod runtime; pub(crate) mod status; mod teardown; @@ -149,12 +152,63 @@ pub(crate) struct Config { pub(crate) uat_ias: u32, /// Output (physical) address bits of the UAT. pub(crate) uat_oas: u32, - /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. - pub(crate) secondary_root_offset: u64, - /// Crash log buffer of each [`Role`]. - pub(crate) crash_buffers: [FixedBuffer; 2], + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region; `None` for + /// a single-instance firmware. + pub(crate) secondary_root_offset: Option, + /// Where the coprocessors' crash log buffers live. + pub(crate) crash_buffers: CrashBuffers, /// Parameter buffer page metrics shared with the firmware. pub(crate) pm_metrics: FixedBuffer, + /// Register windows the firmware maps through the hardware data. + pub(crate) windows: &'static [fw::initdata::RegisterWindow], + /// Register-map slots that are reserved but have no window. + pub(crate) reserved_slots: &'static [usize], + /// Per-SoC parts of the performance tables. + pub(crate) perf: &'static fw::initdata::PerfConfig, + /// Number of operating points the device tree lists. + pub(crate) opps: usize, + /// UAT policy where it differs from the A18 Pro's. + pub(crate) uat_quirks: mmu::UatQuirks, + /// Snapshot the firmware root's own top-level entries after the firmware booted, clear the + /// others and confirm the snapshot after building (A18 Pro). Otherwise the root is used as + /// the firmware left it, which is the only way the M4 Pro has run. + pub(crate) adopt_root_entries: bool, + /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware + /// starts (A18 Pro). The M4 Pro has only run with neither written. + pub(crate) fabric_bridge: bool, + /// The SoC's values of region C. + pub(crate) region_c: &'static fw::initdata::RegionCConfig, + /// Physical address of the firmware's per-slot recovery keys, read while the firmware is + /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the + /// firmware's own blame). + pub(crate) recovery_slot_keys: Option, + /// GPU generation reported to userspace. + pub(crate) gpu_generation: hw::GpuGen, + /// The firmware builds this SoC runs. + pub(crate) firmware: fw::variant::Firmware, + /// Layouts of the command buffers that follow the GPU's core layout. + pub(crate) core_layout: CoreLayout, +} + +/// Layouts of the firmware command buffers that follow a SoC's GPU core layout. +pub(crate) struct CoreLayout { + /// The cluster and enabled core counts the layouts were measured on, which the GPU must + /// report at probe; `None` where the layouts hold for every configuration of the SoC. + pub(crate) cores: Option<(u32, u32)>, + /// A compute queue's operand-state buffer. + pub(crate) operand_state: fw::compute::OperandState, + /// A render's deflake buffer. + pub(crate) deflake: fw::render::Deflake, +} + +/// Where a G17 target's crash log buffers live. +pub(crate) enum CrashBuffers { + /// One buffer per [`Role`] that the firmware requests at a fixed address of the kernel + /// lower root, mapped by the driver before the coprocessors start. + Fixed([FixedBuffer; 2]), + /// The firmware places its crash log in its own data segment, the `fw-data` reserved + /// region, and asks for it there by physical address. + FirmwareData, } /// A firmware coprocessor instance. @@ -197,6 +251,9 @@ const RECEIVER_RETRY_MS: i64 = 20; /// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; +/// One-millisecond polls for a firmware that acknowledges its initialization only in status A +/// ([`fw::variant::InitAck::Status`]). +const INIT_ACCEPT_POLLS: usize = 3000; /// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in /// its buffer requests. @@ -207,7 +264,15 @@ const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; /// A crash log buffer handed to RTKit. pub(crate) struct CrashLog { iova: usize, - map: shmem::VMapOwned, + map: CrashLogMap, +} + +/// The CPU mapping of a crash log buffer. +enum CrashLogMap { + /// The driver's buffer ([`CrashBuffers::Fixed`]). + Object(shmem::VMapOwned), + /// A buffer in the firmware's data segment ([`CrashBuffers::FirmwareData`]). + FirmwareData(data_segment::DataSegmentLog), } impl rtkit::Buffer for CrashLog { @@ -216,7 +281,10 @@ impl rtkit::Buffer for CrashLog { } fn buf(&mut self) -> Result> { - Ok(IoSysMap::from((&self.map).as_view())) + match &mut self.map { + CrashLogMap::Object(map) => Ok(IoSysMap::from((&*map).as_view())), + CrashLogMap::FirmwareData(log) => log.buf(), + } } } @@ -225,8 +293,8 @@ impl rtkit::Buffer for CrashLog { struct RoleState { role: Role, shared: Arc, - crash_log: FixedBuffer, - crash_map: shmem::VMapOwned, + /// The fixed crash log buffer and its mapping; `None` where the firmware places its own. + crash_log: Option<(FixedBuffer, shmem::VMapOwned)>, crashed: AtomicBool, #[pin] ack: Opaque, @@ -305,13 +373,23 @@ impl rtkit::Operations for RoleOps { iova: usize, size: usize, ) -> Result { - if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size - { - return Err(EINVAL); + if let Some((buffer, map)) = data.crash_log.as_ref() { + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != buffer.va || size > buffer.size { + return Err(EINVAL); + } + return Ok(CrashLog { + iova, + map: CrashLogMap::Object(map.owner().owned_vmap()?), + }); } + // The firmware places its buffers in its data segment, at their physical address. Ok(CrashLog { iova, - map: data.crash_map.owner().owned_vmap()?, + map: CrashLogMap::FirmwareData(data_segment::DataSegmentLog::map( + &data.shared.dev, + iova, + size, + )?), }) } } @@ -324,7 +402,7 @@ struct Coprocessor { running: bool, rtkit: rtkit::RtKit, state: Arc, - _crash_buffer: KernelObject, + _crash_buffer: Option, } impl Coprocessor { @@ -338,27 +416,32 @@ impl Coprocessor { role: Role, ) -> Result { let index = role as usize; - let buffer = &cfg.crash_buffers[index]; - let mut crash_buffer = KernelObject::new( - dev, - uat.kernel_lower_vm(), - Placement::At(buffer.va), - buffer.size, - mmu::PROT_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); - - let crash_map = crash_buffer.kernel_map()?; + let (crash_buffer, crash_log) = match &cfg.crash_buffers { + CrashBuffers::Fixed(buffers) => { + let buffer = &buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + let crash_map = crash_buffer.kernel_map()?; + let fixed = FixedBuffer { + va: buffer.va, + size: buffer.size, + }; + (Some(crash_buffer), Some((fixed, crash_map))) + } + CrashBuffers::FirmwareData => (None, None), + }; let state = Arc::pin_init( pin_init!(RoleState { role, shared: shared.clone(), - crash_log: FixedBuffer { - va: buffer.va, - size: buffer.size, - }, - crash_map, + crash_log, crashed: AtomicBool::new(false), ack <- Opaque::ffi_init(|ptr| { // SAFETY: ptr is valid uninitialized completion storage that is pinned @@ -404,8 +487,9 @@ impl Coprocessor { Pin::new(&mut self.rtkit).boot() } - /// Starts the firmware endpoints and hands the firmware its initialization root. - fn init(&mut self, root_va: u64) -> Result { + /// Starts the firmware endpoints and hands the firmware its initialization root. An + /// acknowledgment message is awaited here; one in status A is the caller's to poll. + fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { let mut rtkit = Pin::new(&mut self.rtkit); rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; rtkit.as_mut().start_endpoint(EP_DOORBELL)?; @@ -413,7 +497,10 @@ impl Coprocessor { EP_FIRMWARE, MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), )?; - self.state.wait_for_ack() + match ack { + fw::variant::InitAck::Message => self.state.wait_for_ack(), + fw::variant::InitAck::Status => Ok(()), + } } /// Sends a host notification to the firmware. @@ -452,6 +539,14 @@ impl Bridge { Ok(bridge) } + /// Keeps the register block for a target that runs without the bridge. + fn unused(regs: regs::G17Resources) -> Bridge { + Bridge { + regs, + enabled: false, + } + } + fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -472,15 +567,21 @@ impl Drop for Bridge { /// dropped, the bridge is disabled once they are stopped, and the firmware objects and their /// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. struct Firmware { - secondary: Coprocessor, + /// The secondary instance, on GPUs whose firmware runs on two (A18 Pro). A single-instance + /// firmware (M4 Pro) carries both roles on the primary. + secondary: Option, primary: Coprocessor, bridge: Bridge, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, + hold: power::Hold, registration_power_release_pending: bool, + variant: &'static fw::Variant, queues: runtime::Registry, pool: Arc, + /// Render kick ring storage, kept and reused for the life of the device. + render_rings: Arc, render_ids: Arc, init: initdata::InitData, _render_global: mmu::Vm, @@ -490,7 +591,9 @@ struct Firmware { impl Drop for Firmware { fn drop(&mut self) { - self.secondary.stop(); + if let Some(secondary) = self.secondary.as_mut() { + secondary.stop(); + } self.primary.stop(); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while @@ -500,6 +603,15 @@ impl Drop for Firmware { } impl Firmware { + /// The firmware instances this GPU runs. + fn roles(&self) -> &'static [Role] { + if self.secondary.is_some() { + &[Role::Primary, Role::Secondary] + } else { + &[Role::Primary] + } + } + /// Publishes controls only while the restart handshake is closed. The device mutex remains /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { @@ -519,6 +631,11 @@ impl Firmware { } fn release_registration_power(&mut self) -> Result { + // An idle hold keeps the cores powered past registration; it releases them itself. + if self.hold.active() { + self.registration_power_release_pending = false; + return Ok(()); + } power::release(self) } @@ -548,14 +665,49 @@ impl Firmware { /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, /// unretired work; quarantined work retaining mappings is excluded. fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - match self.effort.idle(now_ns, in_flight) { - power::IdleAction::None => Ok(None), + let effort = match self.effort.idle(now_ns, in_flight) { + power::IdleAction::None => None, power::IdleAction::Clear => { self.apply_effort(0)?; - Ok(None) + None + } + power::IdleAction::Recheck(delay) => Some(delay), + }; + Ok(match (effort, self.idle_hold(now_ns, in_flight)) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }) + } + + /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A + /// refused control (recovery in progress) is retried on a later check. + fn idle_hold(&mut self, now_ns: u64, in_flight: bool) -> Option { + const RETRY_NS: u64 = 1_000_000; + let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); + for _ in 0..2 { + match self.hold.step(now_ns, in_flight, self.effort.submissions(), hold_ms) { + power::HoldAction::None => return None, + power::HoldAction::Recheck(delay) => return Some(delay), + power::HoldAction::Assert => { + if self.set_idle_control(false).is_err() { + return None; + } + self.hold.asserted(); + } + power::HoldAction::Release => { + if self.set_idle_control(true).is_err() { + return Some(RETRY_NS); + } + self.hold.released(); + return None; + } } - power::IdleAction::Recheck(delay) => Ok(Some(delay)), } + None + } + + fn set_idle_control(&mut self, enabled: bool) -> Result { + power::Control::set_idle(self, enabled) } } @@ -610,6 +762,11 @@ struct Shared { drm: Mutex>, clusters: u32, active_cores: u32, + variant: &'static fw::Variant, + /// Layout of compute queues' operand-state buffers for this GPU's cores. + operand_state: &'static fw::compute::OperandState, + /// Layout of renders' deflake buffers for this GPU's cores. + deflake: fw::render::Deflake, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -742,12 +899,20 @@ impl Shared { } fn drain_reclaims(self: &Arc) { + let mut reclaimed = false; loop { let next = self.reclaim.lock().take_one(); let Some(next) = next else { break; }; next.finish(); + reclaimed = true; + } + if reclaimed { + // Context release returns its execution root off-lock. The next + // device pass can now detach idle compute aliases and their QoS + // leases, without depending on a new client's submission/event. + self.queue_events(); } } @@ -826,6 +991,7 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + _capacity: Pin>>, } impl Gpu { @@ -836,6 +1002,7 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; if id.family != cfg.identity.family || id.variant != cfg.identity.variant @@ -860,12 +1027,28 @@ impl Gpu { let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + if cfg + .core_layout + .cores + .is_some_and(|cores| cores != (id.num_clusters, active_cores)) + { + dev_err!( + dev.as_ref(), + "Unsupported GPU core layout: {} clusters with {} cores\n", + id.num_clusters, + active_cores + ); + return Err(ENODEV); + } let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, active_cores, + variant, + operand_state: &cfg.core_layout.operand_state, + deflake: cfg.core_layout.deflake, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, @@ -884,6 +1067,11 @@ impl Gpu { GFP_KERNEL, )?; + let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let capacity = KBox::pin_init( + directory.read_only_file(c"capacity", capacity::View(shared.clone())), + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -892,6 +1080,7 @@ impl Gpu { geometry, ids: Default::default(), shared, + _capacity: capacity, }, GFP_KERNEL, )?; @@ -901,6 +1090,42 @@ impl Gpu { Ok(gpu) } + /// Returns the variant of the loaded firmware build. An identified SoC refuses a build it + /// does not know, before anything is written to the GPU. + fn firmware_variant( + dev: &AsahiDrmDevice, + firmware: &fw::variant::Firmware, + ) -> Result<&'static fw::Variant> { + let builds = match firmware { + fw::variant::Firmware::Fixed(variant) => return Ok(variant), + fw::variant::Firmware::Identified(builds) => builds, + }; + let version = Self::os_firmware_version(dev); + if let Some(build) = builds.iter().find(|build| Some(build.os_version) == version) { + dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + return Ok(build.variant); + } + dev_err!( + dev.as_ref(), + "Unsupported GPU firmware: macOS version {:?} (apple,firmware-version)\n", + version + ); + for build in builds.iter() { + dev_err!(dev.as_ref(), "Supported GPU firmware: {}\n", build.name); + } + Err(ENODEV) + } + + /// The macOS version the boot loader reports for the boot firmware that loaded the GPU + /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. + fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { + let node = dev.as_ref().as_ref().of_node()?; + let cells = node + .get_property::>(c_str!("apple,firmware-version")) + .ok()?; + cells.as_slice().try_into().ok() + } + /// Returns the highest advertised operating-point frequency without changing table order. fn max_frequency_khz(dev: &AsahiDrmDevice) -> Result { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; @@ -932,6 +1157,7 @@ impl Gpu { mmu::UatFirmware::PublishedRoots { secondary_root_offset: cfg.secondary_root_offset, }, + cfg.uat_quirks, )?, GFP_KERNEL, )?; @@ -948,11 +1174,20 @@ impl Gpu { )?; let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; - let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; + let secondary = match cfg.secondary_root_offset { + Some(_) => Some( + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) + })?, + ), + None => None, + }; let render_global = uat.new_vm(0, geometry.user_range())?; - let bridge = Bridge::enable(regs)?; + let bridge = match cfg.fabric_bridge { + true => Bridge::enable(regs)?, + false => Bridge::unused(regs), + }; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. @@ -964,9 +1199,17 @@ impl Gpu { rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), + hold: power::Hold::new(), registration_power_release_pending: false, - queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), + variant: shared.variant, + queues <- runtime::Registry::new( + shared.clusters, + shared.deflake, + shared.descriptor_flags, + shared.variant, + ), pool: object::Pool::new()?, + render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, @@ -977,45 +1220,95 @@ impl Gpu { )?; let session = &mut *firmware; session.primary.boot()?; - session.secondary.boot()?; - let root_entries = session.uat.adopt_firmware_root_entries()?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.boot()?; + } + // The firmware built its root and kernel-window tables while booting. + session.uat.invalidate_reserved_tables(); + let root_entries = match cfg.adopt_root_entries { + true => Some(session.uat.adopt_firmware_root_entries()?), + false => None, + }; session.init.build( dev, cfg, &session.uat, &session._render_global, &session._pm_metrics, + shared.clusters, + shared.active_cores, + shared.variant, )?; - session.uat.confirm_firmware_root_entries(root_entries)?; + if let Some(root_entries) = root_entries { + session.uat.confirm_firmware_root_entries(root_entries)?; + } session.uat.mirror_secondary_root()?; - session.bridge.regs.enable_g17_gating()?; - session.primary.init(session.init.root_va(Role::Primary)?)?; - session - .secondary - .init(session.init.root_va(Role::Secondary)?)?; - session.primary.notify(MSG_POWER_ON_DONE)?; - session.secondary.notify(MSG_POWER_ON_DONE)?; - Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; + if cfg.fabric_bridge { + session.bridge.regs.enable_g17_gating()?; + } + let ack = shared.variant.init_ack; + session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + if ack == fw::variant::InitAck::Status { + Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; + } + if let Some(secondary) = session.secondary.as_mut() { + secondary.init(session.init.root_va(Role::Secondary)?, ack)?; + } + if shared.variant.power_on_done { + session.primary.notify(MSG_POWER_ON_DONE)?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.notify(MSG_POWER_ON_DONE)?; + } + } + Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; session .uat .activate_render_context(&session._render_global)?; Ok(firmware) } + /// Waits for a firmware that sends no acknowledgment message to accept its + /// initialization in status A. + fn wait_for_init_accepted( + dev: &AsahiDrmDevice, + init: &initdata::InitData, + primary: &Coprocessor, + ) -> Result { + for poll in 0..=INIT_ACCEPT_POLLS { + if primary.state.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if init.init_accepted()? { + return Ok(()); + } + if poll < INIT_ACCEPT_POLLS { + fsleep(Delta::from_millis(1)); + } + } + dev_err!( + dev.as_ref(), + "Firmware did not accept its initialization: status A {:#x?}\n", + init.status_read_words::<4>(Role::Primary, 0x10)? + ); + Err(ETIMEDOUT) + } + /// Waits for both coprocessors to retire their opening device-control records. fn wait_for_opening( dev: &AsahiDrmDevice, init: &initdata::InitData, primary: &Coprocessor, - secondary: &Coprocessor, + secondary: Option<&Coprocessor>, ) -> Result { for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) - || secondary.state.crashed.load(Ordering::Acquire) + || secondary.is_some_and(|s| s.state.crashed.load(Ordering::Acquire)) { return Err(EIO); } - if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { + if init.opening_retired(Role::Primary)? + && (secondary.is_none() || init.opening_retired(Role::Secondary)?) + { return Ok(()); } if poll < OPENING_RETIRE_POLLS { @@ -1081,12 +1374,20 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= - uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY as u64; - params.gpu_generation = hw::GpuGen::G17 as u32; + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + let variant = self.shared.variant; + if variant.few_primitives { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + } + if variant.ray_traversal { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); + } + params.gpu_generation = self.cfg.gpu_generation as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; params.chip_id = self.cfg.chip_id; @@ -1115,9 +1416,18 @@ impl gpu::Gpu for Gpu { true } + /// The G17 runtime never programs fault_control into the firmware, so unmapped reads are not + /// guaranteed to return zero. Advertising soft faults made Mesa point out-of-bounds vec3/vec4 + /// vertex fetches at address 0 and skip bounds checks on bounded loads; on T6040 those reads + /// returned non-zero data (2026-10-19: every robustness2 rgba32 vertex_attribute_fetch case + /// failed; all pass with soft faults off). + fn supports_soft_faults(&self) -> bool { + false + } + fn queue_limits(&self) -> Option { Some(uapi::drm_asahi_queue_limits { - max_queues: mmu::MAX_EXECUTION_CONTEXTS, + max_queues: kick::queue_limit(self.shared.variant.qid_floor), max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, ..Default::default() }) @@ -1156,12 +1466,26 @@ impl gpu::Gpu for Gpu { ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); - let frontend = - queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let frontend = queue::Frontend::new( + priority, + usc_exec_base, + self.geometry.user_usable_range(), + self.shared.variant, + )?; let policy = scheduling_policy(priority)?; + let admission_started = now_ns(); let (backend, slot) = loop { match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, + Err(error) if error == ENOSPC => { + // Closed logical queues can still own execution/QoS IDs + // until the scheduler release is consumed and reclaimed. + // Backend construction has rolled back all temporary leases. + if now_ns().saturating_sub(admission_started) + >= job::TIMEOUT_MS as u64 * 1_000_000 { return Err(EBUSY); } + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } result => break result?, } }; @@ -1185,27 +1509,28 @@ impl gpu::Gpu for Gpu { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { + ) -> Result { let state = self.shared.state.lock(); let firmware = (*state).as_ref().ok_or(ENODEV)?; let mut addresses = firmware.uat.geometry().kernel_range(); addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - bo.map_range_into_range( + crate::timestamp::Buffer::firmware(bo.map_range_into_range( firmware.uat.kernel_vm(), range, addresses, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - ) + )?) } } /// Scheduling profile of a new logical queue. `file.rs` passes -/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses HIGH and REALTIME, so -/// MEDIUM arrives as 2 and LOW as 3. +/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses REALTIME, so HIGH +/// arrives as 1, MEDIUM as 2 and LOW as 3. fn scheduling_policy(priority: u32) -> Result { match priority { + 1 => Ok(hw::t8140::scheduling::HIGH), 2 => Ok(hw::t8140::scheduling::MEDIUM), 3 => Ok(hw::t8140::scheduling::LOW), _ => Err(EINVAL), diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index 46f4e0c56b6ae..f1b22ad5d7331 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -78,7 +78,17 @@ impl KernelObject { let mut this = Self::backing(dev, size, cpu)?; let mapping = match placement { Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, - Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, + Placement::In(range, align) => { + let site = if range.start >= crate::hw::t8140::dynamic::LOWER.start + && range.end <= crate::hw::t8140::dynamic::LOWER.end { + super::capacity::Site::LowerVa + } else { + super::capacity::Site::UpperVa + }; + this.obj.map_into_range(vm, range, align, prot, true).map_err(|error| { + if error == ENOSPC { super::capacity::refused(site) } else { error } + })? + } }; this.mapping = Some(mapping); Ok(this) @@ -113,6 +123,9 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) + .map_err(|error| if error == ENOSPC { + super::capacity::refused(super::capacity::Site::AliasVa) + } else { error }) } /// Reuses only a complete alias of this exact retained backing. Permanent @@ -159,6 +172,20 @@ impl KernelObject { prot: mmu::Prot, ) -> Result { gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) + .map_err(|error| if error == ENOSPC { + super::capacity::refused(super::capacity::Site::AliasVa) + } else { error }) + } + + fn alias_in_lowest( + &self, + vm: &mmu::Vm, + range: Range, + align: u64, + prot: mmu::Prot, + guard_size: usize, + ) -> Result { + vm.map_in_range_lowest(&self.obj.gem, 0..self.size, align, range, prot, guard_size) } /// Maps an exact backing subrange, retaining an explicit unmapped guard. @@ -341,6 +368,76 @@ impl Allocator<'_> { cpu, ) } + + /// Place fresh backing at one address in the canonical and both alias VMs. + /// A client may retain mappings after their canonical reservation is freed. + /// Retry only address collisions, keeping the same backing throughout. + pub(crate) fn lower_with_aliases( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + aliases: [&mmu::Vm; 2], + ) -> Result<(KernelObject, [mmu::KernelMapping; 2])> { + if size == 0 || size % mmu::UAT_PGSZ != 0 { + return Err(EINVAL); + } + let mut object = KernelObject::backing(self.dev, size, cpu)?; + let mut range = crate::hw::t8140::dynamic::LOWER; + let mut searching = false; + loop { + // Keep ordinary best-fit placement on the uncontended path. A + // collision starts a lowest-address search over the whole arena. + let mapping = (if searching { + object.alias_in_lowest( + self.uat.kernel_lower_vm(), range.clone(), align, prot, mmu::UAT_PGSZ, + ) + } else { + object.obj.map_into_range( + self.uat.kernel_lower_vm(), range.clone(), align, prot, true, + ) + }).map_err(|error| if error == ENOSPC { + crate::g17::capacity::refused(crate::g17::capacity::Site::LowerVa) + } else { error })?; + let address = mapping.iova(); + let end = address.checked_add(size as u64).ok_or(EOVERFLOW)?; + // Partial aliases are destroyed before releasing the canonical + // reservation, so a retry cannot expose its address prematurely. + let mapped: core::result::Result<_, (Error, usize)> = (|| { + let first = object.map_alias(aliases[0], address, prot) + .map_err(|error| (error, 0))?; + let second = object.map_alias(aliases[1], address, prot) + .map_err(|error| (error, 1))?; + Ok([first, second]) + })(); + match mapped { + Ok(mapped) => { + object.mapping = Some(mapping); + return Ok((object, mapped)); + } + Err((error, index)) if error == ENOSPC && end < range.end => { + drop(mapping); + let start = if searching { + address + mmu::UAT_PGSZ as u64 + } else { + range.start + }; + // Jump to an actual hole in the VM that refused the alias, + // rather than remapping backing at every intervening page. + let candidate = object.alias_in_lowest( + aliases[index], start..range.end, align, prot, 0, + ).map_err(|error| if error == ENOSPC { + crate::g17::capacity::refused(crate::g17::capacity::Site::AliasVa) + } else { error })?; + range.start = candidate.iova(); + drop(candidate); + searching = true; + } + Err((error, _)) => return Err(error), + } + } + } } /// Families whose upper mappings remain cached and installed until firmware @@ -577,3 +674,16 @@ impl Pool { } } } + +impl Pool { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let inner = self.inner.lock(); + for family in 0..POOL_FAMILIES { + let entries = &inner.returned[family * POOL_SLOTS..(family + 1) * POOL_SLOTS]; + writeln!(f, "cached_objects_{} used={} total={} returned={} poisoned={}", family, + inner.reserved[family], POOL_SLOTS, entries.iter().flatten().count(), + entries.iter().flatten().filter(|entry| entry.poisoned).count())?; + } + writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES) + } +} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 96b363bc27447..7fef15084c50a 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -3,9 +3,19 @@ //! Work-based firmware performance effort. Idle power gating is managed separately by the //! device-control ring; this input never keeps an otherwise idle GPU powered. +use core::sync::atomic::AtomicU32; + pub(super) const FULL_EFFORT: u32 = 0x10000; const IDLE_HOLD_NS: u64 = 16_000_000; +/// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after +/// the last job. Not measured on other firmware; at or below it the hold stays off. +pub(crate) const FIRMWARE_IDLE_OFF_MS: u32 = 2; + +/// The `idle_off_delay_ms` sysfs knob. Above [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU +/// cores powered until the GPU has been idle this long ([`Hold`]). +pub(crate) static IDLE_HOLD_MS: AtomicU32 = AtomicU32::new(FIRMWARE_IDLE_OFF_MS); + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum IdleAction { None, @@ -17,6 +27,7 @@ pub(super) struct Effort { streaming: bool, idle_since: Option, applied: u32, + submissions: u64, } impl Effort { @@ -25,12 +36,19 @@ impl Effort { streaming: false, idle_since: None, applied: 0, + submissions: 0, } } pub(super) fn submit(&mut self) { self.streaming = true; self.idle_since = None; + self.submissions = self.submissions.wrapping_add(1); + } + + /// Count of published submissions, for [`Hold::step`]. + pub(super) fn submissions(&self) -> u64 { + self.submissions } pub(super) fn applied(&self) -> u32 { @@ -62,6 +80,83 @@ impl Effort { } } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum HoldAction { + None, + /// Send the keep-powered idle policy. + Assert, + /// Send the power-off idle policy again. + Release, + Recheck(u64), +} + +/// Host-side idle hold. On T6040/25G83 the firmware powers the cores off 2 ms after the last job +/// and reads that delay only at boot; every wake-up then costs about 1 ms. With a longer +/// [`IDLE_HOLD_MS`] the host sends the keep-powered idle policy once work appears and the power-off +/// policy once no work has been submitted or in flight for that long. Checked from the event +/// worker, which runs on every completion, well inside the firmware's 2 ms. +pub(super) struct Hold { + active: bool, + seen: u64, + since: Option, +} + +impl Hold { + pub(super) const fn new() -> Self { + Self { + active: false, + seen: 0, + since: None, + } + } + + pub(super) fn active(&self) -> bool { + self.active + } + + pub(super) fn step( + &mut self, + now_ns: u64, + in_flight: bool, + submissions: u64, + hold_ms: u32, + ) -> HoldAction { + let enabled = hold_ms > FIRMWARE_IDLE_OFF_MS; + let fresh = submissions != self.seen; + self.seen = submissions; + if in_flight || fresh { + self.since = None; + } + if !self.active { + return if enabled && (in_flight || fresh) { + HoldAction::Assert + } else { + HoldAction::None + }; + } + if in_flight { + return HoldAction::None; + } + let hold_ns = if enabled { u64::from(hold_ms) * 1_000_000 } else { 0 }; + let since = *self.since.get_or_insert(now_ns.max(1)); + let idle = now_ns.saturating_sub(since); + if idle >= hold_ns { + HoldAction::Release + } else { + HoldAction::Recheck(hold_ns - idle) + } + } + + pub(super) fn asserted(&mut self) { + self.active = true; + } + + pub(super) fn released(&mut self) { + self.active = false; + self.since = None; + } +} + /// Single-producer control path under the device mutex. Event draining must not publish another /// control between `set_idle` and the exact three-counter retirement witness. pub(super) trait Control { @@ -137,3 +232,31 @@ pub(super) fn release(control: &mut C) -> kernel::error::Result { } result } + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_idle_hold))] +mod tests { + use super::*; + + #[test] + fn holds_until_idle_for_the_delay_then_releases() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, 10), HoldAction::Assert); + hold.asserted(); + assert_eq!(hold.step(1_000_000, false, 1, 10), HoldAction::Recheck(10_000_000)); + assert_eq!(hold.step(6_000_000, false, 1, 10), HoldAction::Recheck(5_000_000)); + // A new submission restarts the idle period. + assert_eq!(hold.step(7_000_000, false, 2, 10), HoldAction::Recheck(10_000_000)); + assert_eq!(hold.step(17_000_000, false, 2, 10), HoldAction::Release); + hold.released(); + assert!(!hold.active()); + } + + #[test] + fn the_default_delay_never_holds_and_lowering_it_releases() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, FIRMWARE_IDLE_OFF_MS), HoldAction::None); + assert_eq!(hold.step(0, true, 2, 10), HoldAction::Assert); + hold.asserted(); + assert_eq!(hold.step(1, false, 2, FIRMWARE_IDLE_OFF_MS), HoldAction::Release); + } +} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 8563beda979e6..0297c619b4f93 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -20,6 +20,34 @@ const REMOVED: u64 = CLOSED << 1; const FAILED: u64 = CLOSED << 2; const EPOCH: u64 = CLOSED << 3; +/// One ready packet's monotonic compute-admission budget. The callback checks +/// packet cancellation, logical close, VM status and device loss even when no +/// capacity/recovery event arrives. Wakes and nested constructors never renew it. +pub(super) struct Admission<'a> { + deadline: u64, + check: &'a dyn Fn() -> Result, +} + +impl<'a> Admission<'a> { + pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { + Self { deadline, check } + } + + pub(super) fn check(&self) -> Result { + (self.check)()?; + if super::now_ns() >= self.deadline { return Err(EBUSY); } + Ok(()) + } + + /// Ignore condvar remaining-time reports: only the monotonic deadline counts. + pub(super) fn slice_ms(&self) -> Result { + self.check()?; + let remaining = self.deadline.saturating_sub(super::now_ns()); + if remaining == 0 { return Err(EBUSY); } + Ok(remaining.div_ceil(1_000_000).min(10) as u32) + } +} + #[pin_data] pub(super) struct Gate { state: AtomicU64, @@ -92,6 +120,27 @@ impl Gate { } } + /// Packet admission may wait for recovery only within its original budget. + /// Called with no device mutex or existing preparation lease. + pub(super) fn enter_until(self: &Arc, admission: &Admission<'_>) -> Result { + admission.check()?; + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let mut wait = self.wait.lock(); + loop { + admission.check()?; + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let _ = self.changed.wait_timeout( + &mut wait, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); + } + } + fn invalidate(&self, flags: u64) { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -129,6 +178,17 @@ impl Gate { self.changed.notify_all(); } + /// Runtime failure/recovery must not join an indefinitely stalled builder. + /// Timeout retains every graph and reservation; it never authorizes freeing. + pub(super) fn wait_drained_until(&self, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { + if super::now_ns() >= deadline { return Err(ETIMEDOUT); } + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(10)); + } + Ok(()) + } + /// Join builders with no device, queue or mapping lock held. pub(super) fn wait_drained(&self) { let mut wait = self.wait.lock(); @@ -184,3 +244,96 @@ impl Drop for Lease { } } } + +#[kunit_tests(asahi_g17_growth_gate)] +mod growth_gate_tests { + use super::*; + #[test] + fn recovery_can_join_between_growth_admissions() { + let gate = Gate::new().unwrap(); + let outer = gate.try_enter().unwrap(); + // The growth handoff must drop this lease before enter can wait. + drop(outer); + gate.close(); + assert!(matches!(gate.try_enter(), Err(error) if error == EAGAIN)); + gate.wait_drained(); + assert!(gate.reopen()); + let growth = gate.enter(false).unwrap(); + assert!(growth.is_current()); + drop(growth); + } +} + +#[kunit_tests(asahi_g17_compute_deadline)] +mod compute_deadline_tests { + use super::*; + + #[test] + fn spent_budget_refuses_even_an_open_gate() { + let gate = Gate::new().unwrap(); + let check = || Ok(()); + let admission = Admission::new(super::super::now_ns(), &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn closed_gate_expires_without_any_recovery_wake() { + let gate = Gate::new().unwrap(); + gate.close(); + let start = super::super::now_ns(); + let check = || Ok(()); + let admission = Admission::new(start + 20_000_000, &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert!(super::super::now_ns() - start < 500_000_000); + gate.wait_drained(); + assert!(gate.reopen()); + } + + #[test] + fn cancellation_without_capacity_or_recovery_events() { + let gate = Gate::new().unwrap(); + gate.close(); + let start = super::super::now_ns(); + let check = || { + if super::super::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } + }; + let admission = Admission::new(start + 2_000_000_000, &check); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); + assert!(super::super::now_ns() - start < 500_000_000); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn recovery_and_binding_retry_churn_cannot_renew_budget() { + let gate = Gate::new().unwrap(); + let start = super::super::now_ns(); + let check = || Ok(()); + let admission = Admission::new(start + 20_000_000, &check); + loop { + // Re-entry models a channel taken between admission/publication. + // Recovery joins each dropped lease, then wakes the same budget. + gate.close(); + gate.changed.notify_all(); + gate.wait_drained(); + assert!(gate.reopen()); + match gate.enter_until(&admission) { + Ok(lease) => drop(lease), + Err(error) => { assert_eq!(error, EBUSY); break; } + } + } + assert!(super::super::now_ns() - start < 500_000_000); + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } + + #[test] + fn packet_status_and_cancel_precede_available_capacity() { + let gate = Gate::new().unwrap(); + for error in [ECANCELED, EIO] { + let check = || Err(error); + let admission = Admission::new(u64::MAX, &check); + assert!(matches!(gate.enter_until(&admission), Err(got) if got == error)); + } + assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 1c0d6822c395f..ebdc79d2b2dd3 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -47,12 +47,25 @@ struct InFlight { const QUARANTINE_WAIT_MS: u32 = 1000; +/// A wake never renews admission's budget. Short observations also notice a +/// packet cancellation that has no corresponding pair-release notification. +fn wait_slice_ms(deadline_ns: u64, now_ns: u64) -> Result { + let remaining = deadline_ns.saturating_sub(now_ns); + if remaining == 0 { + return Err(EBUSY); + } + Ok(remaining.div_ceil(1_000_000).min(10) as u32) +} + struct State { active: u64, refs: [u8; cfg::RENDER_SLOTS], owners: [u64; cfg::RENDER_SLOTS], epochs: [u64; cfg::RENDER_SLOTS], quarantined: u64, + /// Quarantined slots whose failed work is all spared (innocent): waiting publications fail + /// their own fence with EBUSY instead of a device-loss error, so the VM stays usable. + spared: u64, kill_hold: u64, released: u64, epoch: u64, @@ -68,6 +81,7 @@ impl State { owners: [0; cfg::RENDER_SLOTS], epochs: [0; cfg::RENDER_SLOTS], quarantined: 0, + spared: 0, kill_hold: 0, released: 0, epoch: 0, @@ -92,7 +106,7 @@ impl State { & Self::MASK == 0 => { - Err(ENOSPC) + Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RenderSlots)) } _ => Ok(()), } @@ -115,18 +129,20 @@ impl State { } // Prefer the owner's idle pair. A missing sibling can be claimed // only while this owner has no live kick, before constructing it. - for slot in 0..cfg::RENDER_SLOTS { - if self.owners[slot] == owner - && self.refs[slot] == 0 - && self.quarantined & (1u64 << slot) == 0 + let idle = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.refs[*slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + }) + .max_by_key(|slot| (self.epochs[*slot], usize::MAX - *slot)); + if let Some(slot) = idle { + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) { - if !(0..cfg::RENDER_SLOTS) - .any(|other| self.owners[other] == owner && self.refs[other] != 0) - { - self.claim_sibling(owner); - } - return Some(self.take_idle(slot, owner)); + self.claim_sibling(owner); } + return Some(self.take_idle(slot, owner)); } if self.owners.contains(&owner) { // Consecutive kicks are ordered by firmware dependencies. Stack @@ -175,9 +191,52 @@ impl State { if let Some(slot) = sibling { self.released &= !(1u64 << slot); self.owners[slot] = owner; - self.advance(slot); + // A speculative sibling has no GPU-use recency yet. + self.epochs[slot] = 0; } } + + /// The device lock separately proves the physical graph was never published. + /// An open owner keeps another usable pair. Exclude every live borrower, + /// including off-lock sibling construction; a closed owner needs no spare. + fn reassign_unused_spare( + &mut self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + let slot = usize::from(slot); + if slot == 0 + || slot >= cfg::RENDER_SLOTS + || owner.is_some_and(|owner| { + owner == 0 + || self.owners.iter().filter(|value| **value == owner).count() + >= cfg::RENDER_PAIRS_PER_OWNER + }) + { + return false; + } + let bit = 1u64 << slot; + let old = self.owners[slot]; + if old == 0 + || owner == Some(old) + || expected.is_some_and(|expected| expected != old) + || self.refs[slot] != 0 + || (self.active | self.quarantined | self.kill_hold) & bit != 0 + || (self.released & bit == 0 + && !self.owners.iter().enumerate().any(|(other, value)| { + other != slot + && *value == old + && (self.quarantined | self.kill_hold) & (1u64 << other) == 0 + })) + { + return false; + } + self.owners[slot] = owner.unwrap_or(0); + self.released &= !bit; + self.epochs[slot] = 0; + true + } fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { if self.refs.get(usize::from(publishing)) != Some(&1) { return None; @@ -188,9 +247,31 @@ impl State { && self.owners[*slot] == owner && self.refs[*slot] == 0 && self.quarantined & (1u64 << slot) == 0 + && self.released & (1u64 << slot) == 0 }) .map(|slot| slot as u8) } + fn borrow_warm_sibling(&mut self, owner: u64, publishing: u8) -> Option { + let slot = self.warmable_sibling(owner, publishing)?; + self.refs[usize::from(slot)] += 1; + self.active |= 1u64 << slot; + Some(slot) + } + fn cancel_empty_slot(&mut self, slot: u8, owner: u64) -> bool { + let slot = usize::from(slot); + let bit = 1u64 << slot; + if self.owners[slot] == owner + && self.refs[slot] == 1 + && (self.quarantined | self.kill_hold) & bit == 0 + { + self.owners[slot] = 0; + self.released &= !bit; + self.epochs[slot] = 0; + true + } else { + false + } + } fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { let Some(request) = request else { return Plan::Any; @@ -283,23 +364,33 @@ pub(crate) struct Pool { #[pin] released: CondVar, queues: AtomicU32, + /// Logical queues admitted at most: each must be able to hold a render pair. + max_queues: u32, } impl Pool { - pub(crate) fn new() -> Result> { + pub(crate) fn new(max_queues: u32) -> Result> { + if max_queues > MAX_EXECUTION_CONTEXTS { + return Err(EINVAL); + } Arc::pin_init( pin_init!(Self { state <- new_mutex!(State::new(), "G17 render admission"), released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), + max_queues, }), GFP_KERNEL, ) } + /// Logical queues currently counted, including one being created. + pub(crate) fn queues(&self) -> u32 { + self.queues.load(Ordering::Acquire) + } pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { - if count >= MAX_EXECUTION_CONTEXTS { - return Err(ENOSPC); + if count >= self.max_queues { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::LogicalQueues)); } match self.queues.compare_exchange( count, @@ -316,16 +407,22 @@ impl Pool { self: &Arc, owner: u64, request: Option>, + deadline_ns: u64, + check: impl Fn() -> Result, ) -> Result { if owner == 0 { return Err(EINVAL); } let mut state = self.state.lock(); - let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); - let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); + let started = crate::g17::now_ns(); + let conflict_deadline = started.saturating_add(u64::from(hazard::WAIT_MS) * 1_000_000); + let quarantine_deadline = started.saturating_add(u64::from(QUARANTINE_WAIT_MS) * 1_000_000); let mut expired = false; let mut ticket = None; let result = (|| loop { + check()?; + let now = crate::g17::now_ns(); + let remaining_ms = wait_slice_ms(deadline_ns, now)?; if state.exhausted() { return Err(EIO); } @@ -337,20 +434,26 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - if quarantine_budget == 0 { - return Err(EIO); + if now >= quarantine_deadline { + // Behind an innocent pair's quarantine (another VM wedged the GPU) only this + // publication fails, not the VM: it recovers once the pair drains. + let spared = state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.spared & (1u64 << slot) != 0); + return Err(if spared { EBUSY } else { EIO }); + } + let slice = remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?); + if matches!(self.released.wait_interruptible_timeout( + &mut state, kernel::time::msecs_to_jiffies(slice) + ), CondVarTimeoutResult::Signal { .. }) { + return Err(ERESTARTSYS); } - quarantine_budget = match self - .released - .wait_interruptible_timeout(&mut state, quarantine_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, - }; continue; } state.refusal(owner)?; + expired |= now >= conflict_deadline; let plan = if expired { Plan::Any } else { @@ -376,18 +479,17 @@ impl Pool { } } } - match self - .released - .wait_interruptible_timeout(&mut state, conflict_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => { - conflict_budget = 0; - expired = true; - } - CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, - } - } else if self.released.wait_interruptible(&mut state) { + } + // Keep the established hazard wait/fallback rule, but cap every + // sleep with the caller's original deadline and cancellation poll. + let slice = if plan != Plan::Any { + remaining_ms.min(wait_slice_ms(conflict_deadline, now)?) + } else { + remaining_ms + }; + if matches!(self.released.wait_interruptible_timeout( + &mut state, kernel::time::msecs_to_jiffies(slice) + ), CondVarTimeoutResult::Signal { .. }) { return Err(ERESTARTSYS); } })(); @@ -420,8 +522,33 @@ impl Pool { record, })) } - pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { - self.state.lock().warmable_sibling(owner, publishing) + pub(crate) fn warmable_sibling( + self: &Arc, + owner: u64, + publishing: u8, + ) -> Option { + let mut state = self.state.lock(); + let slot = state.borrow_warm_sibling(owner, publishing)?; + // This is a host construction borrower, not a GPU-use epoch. In + // particular it must not clear a logical close or age an unused spare. + Some(RenderLease { + pool: self.clone(), + slot, + record: None, + }) + } + pub(crate) fn reassign_unused_spare( + &self, + slot: u8, + expected: Option, + owner: Option, + ) -> bool { + self.state + .lock() + .reassign_unused_spare(slot, expected, owner) + } + pub(crate) fn owns_slot(&self, owner: u64, slot: usize) -> bool { + self.state.lock().owners.get(slot) == Some(&owner) } pub(crate) fn release_owner(&self, owner: u64) { let mut state = self.state.lock(); @@ -446,12 +573,82 @@ impl Pool { } state.released &= !released; state.quarantined &= !released; + state.spared &= !released; state.kill_hold = 0; drop(state); self.released.notify_all(); } } +#[kunit_tests(asahi_g17_admission_accounting)] +mod accounting_tests { + use super::*; + + #[test] + fn every_logical_owner_fits_without_speculative_spares() { + let mut state = State::new(); + for owner in 1..=u64::from(MAX_EXECUTION_CONTEXTS) { + let slot = state.acquire(owner).unwrap(); + assert_eq!(state.owners.iter().filter(|id| **id == owner).count(), 1); + state.release(slot); + } + assert_eq!(state.active, 0); + assert_eq!(state.owners.iter().filter(|id| **id != 0).count(), MAX_EXECUTION_CONTEXTS as usize); + } + + #[test] + fn close_does_not_reuse_an_active_lease() { + let mut state = State::new(); + let slot = state.acquire(1).unwrap(); + state.released |= 1u64 << slot; + assert!(!state.reassign_unused_spare(slot, Some(1), None)); + state.release(slot); + assert!(state.reassign_unused_spare(slot, Some(1), None)); + assert_eq!(state.owners[usize::from(slot)], 0); + assert_eq!(state.refs[usize::from(slot)], 0); + assert_eq!(state.active & (1u64 << slot), 0); + } + + #[test] + fn warming_quarantine_and_exit_hold_exclude_reassignment() { + let mut state = State::new(); + let primary = state.acquire(1).unwrap(); + // A retained legacy sibling still obeys all quarantine/kill guards. + state.owners[2] = 1; + let spare = state.borrow_warm_sibling(1, primary).unwrap(); + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.release(spare); + state.quarantined |= 1u64 << spare; + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.quarantined = 0; + state.kill_hold |= 1u64 << spare; + assert!(!state.reassign_unused_spare(spare, Some(1), Some(2))); + state.kill_hold = 0; + assert!(!state.reassign_unused_spare(spare, Some(3), Some(2))); + assert!(state.reassign_unused_spare(spare, Some(1), Some(2))); + assert!(!state.reassign_unused_spare(primary, Some(1), None)); + state.release(primary); + assert!(!state.reassign_unused_spare(primary, Some(1), None)); + } + + #[test] + fn logical_queue_count_returns_on_drop() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let mut leases = KVec::with_capacity(MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..MAX_EXECUTION_CONTEXTS { + leases.push(pool.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + assert!(matches!(pool.reserve_queue(), Err(ENOSPC))); + assert_eq!(pool.queues.load(Ordering::Acquire), MAX_EXECUTION_CONTEXTS); + drop(leases.pop()); + let replacement = pool.reserve_queue().unwrap(); + drop(leases); + assert_eq!(pool.queues.load(Ordering::Acquire), 1); + drop(replacement); + assert_eq!(pool.queues.load(Ordering::Acquire), 0); + } +} + /// Counted once per userspace logical queue, independently of installed graphs. pub(crate) struct QueueSlot { pool: Arc, @@ -468,17 +665,40 @@ pub(crate) struct RenderLease { record: Option<(u64, u64)>, } impl RenderLease { + /// The device lock proves this slot has no physical identities or graph. + /// Give back an unbound reservation before retrying on a retained channel. + pub(crate) fn cancel_empty(self, owner: u64) -> bool { + let cancelled = { + let mut state = self.pool.state.lock(); + state.cancel_empty_slot(self.slot, owner) + }; + drop(self); + cancelled + } + pub(crate) fn slot(&self) -> u8 { self.slot } - pub(crate) fn quarantine(&self) { - self.pool.state.lock().quarantined |= 1u64 << self.slot; + /// `spared`: the failing work is innocent. Any guilty failure makes the slot's quarantine + /// guilty for good; only a quarantine that starts spared and stays spared is marked spared. + pub(crate) fn quarantine(&self, spared: bool) { + let bit = 1u64 << self.slot; + let mut state = self.pool.state.lock(); + if !spared { + state.spared &= !bit; + } else if state.quarantined & bit == 0 { + state.spared |= bit; + } + state.quarantined |= bit; + drop(state); self.pool.released.notify_all(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. pub(crate) fn rehabilitate_after_processor_stop(&self) { - self.pool.state.lock().quarantined &= !(1u64 << self.slot); + let mut state = self.pool.state.lock(); + state.quarantined &= !(1u64 << self.slot); + state.spared &= !(1u64 << self.slot); } /// Requires the runtime's closed-recovery, exact pair-drain witness. pub(crate) fn rehabilitate_after_firmware_drain(&self) { @@ -539,3 +759,169 @@ impl Pool { self.released.notify_all(); } } + +impl Pool { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), self.max_queues)?; + let state = self.state.lock(); + let retained = state.owners.iter().enumerate().fold(0u64, |bits, (slot, owner)| { + bits | if *owner != 0 { 1u64 << slot } else { 0 } + }) & State::MASK; + let used = (state.active | state.quarantined | state.kill_hold | (retained & !state.released)) & State::MASK; + writeln!(f, "render_slots used={} total={} retained={} active={} quarantined={} kill_held={}", + used.count_ones(), State::MASK.count_ones(), retained.count_ones(), + (state.active & State::MASK).count_ones(), (state.quarantined & State::MASK).count_ones(), + (state.kill_hold & State::MASK).count_ones()) + } +} + +#[kunit_tests(asahi_g17_fault_admission)] +mod fault_capacity_tests { + use super::*; + + #[test] + fn a_guilty_failure_makes_a_spared_quarantine_guilty() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let queue = pool.reserve_queue().unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let bit = 1u64 << lease.slot(); + lease.quarantine(true); + assert_ne!(pool.state.lock().spared & bit, 0); + lease.quarantine(false); + assert_eq!(pool.state.lock().spared & bit, 0); + // Once guilty, a later spared failure does not make it spared again. + lease.quarantine(true); + assert_eq!(pool.state.lock().spared & bit, 0); + assert_ne!(pool.state.lock().quarantined & bit, 0); + pool.release_owner(1); + lease.rehabilitate_after_firmware_drain(); + drop(lease); + drop(queue); + let state = pool.state.lock(); + assert_eq!(state.quarantined | state.spared, 0); + } + + #[test] + fn closed_faulting_clients_return_leases_after_drain() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + for owner in 1..=512 { + let queue = pool.reserve_queue().unwrap(); + let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + lease.quarantine(false); + pool.release_owner(owner); + let bit = 1u64 << lease.slot(); + assert_ne!(pool.state.lock().active & bit, 0); + assert_ne!(pool.state.lock().quarantined & bit, 0); + // The runtime, not an error fence, supplies this exact drain witness. + lease.rehabilitate_after_firmware_drain(); + drop(lease); + drop(queue); + let state = pool.state.lock(); + assert_eq!(state.active, 0); + assert_eq!(state.quarantined, 0); + assert_eq!(state.owners.iter().enumerate().filter(|(slot, id)| **id != 0 && state.released & (1u64 << slot) == 0).count(), 0); + assert_eq!(pool.queues.load(Ordering::Acquire), 0); + } + } + + #[test] + fn published_exit_kill_holds_survive_close_and_drain() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let bit = 1u64 << lease.slot(); + pool.release_owner(1); + drop(lease); + assert!(pool.hold_render_slots_for_exit_kill(1, bit)); + assert!(!pool.reassign_unused_spare(bit.trailing_zeros() as u8, Some(1), Some(2))); + // A failed (unpublished) control may roll back its hold. Published kills + // keep the existing processor-stop rule; this test must not weaken it. + pool.release_retained_owners_after_processor_stop(); + assert_eq!(pool.state.lock().kill_hold, 0); + } +} + +impl Pool { + pub(crate) fn owner_active(&self, owner: u64) -> bool { + let state = self.state.lock(); + state.owners.iter().zip(state.refs.iter()).any(|(id, refs)| *id == owner && *refs != 0) + } +} + +#[kunit_tests(asahi_g17_shared_render_depth)] +mod shared_depth_tests { + use super::*; + #[test] + fn a_full_pair_waits_for_retirement_instead_of_allocating_a_spare() { + let mut state = State::new(); + let first = state.acquire(1).unwrap(); + for _ in 1..cfg::RENDER_DEPTH { assert_eq!(state.acquire(1), Some(first)); } + assert_eq!(state.acquire(1), None); + assert_eq!(state.owners.iter().filter(|id| **id == 1).count(), 1); + state.release(first); + assert_eq!(state.acquire(1), Some(first)); + } +} + +#[kunit_tests(asahi_g17_admission_deadline)] +mod deadline_tests { + use super::*; + + #[test] + fn all_pairs_busy_no_progress_expires_without_release() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + // No pair has room, including the requesting owner's retained pair. + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = slot as u64; + state.refs[slot] = cfg::RENDER_DEPTH; + } + state.active = State::MASK; + } + let start = crate::g17::now_ns(); + let result = pool.acquire_render(1, None, start + 20_000_000, || Ok(())); + assert!(matches!(result, Err(error) if error == EBUSY)); + assert!(crate::g17::now_ns() - start < 500_000_000); + assert!(pool.state.lock().footprints.is_empty()); + } + + #[test] + fn busy_pair_observes_cancellation_without_a_release_event() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = slot as u64; + state.refs[slot] = cfg::RENDER_DEPTH; + } + state.active = State::MASK; + } + let start = crate::g17::now_ns(); + let result = pool.acquire_render(1, None, start + 2_000_000_000, || { + if crate::g17::now_ns() - start >= 10_000_000 { Err(ECANCELED) } else { Ok(()) } + }); + assert!(matches!(result, Err(error) if error == ECANCELED)); + assert!(crate::g17::now_ns() - start < 500_000_000); + assert!(pool.state.lock().footprints.is_empty()); + } + + #[test] + fn cancellation_precedes_available_capacity() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), + Err(error) if error == ECANCELED)); + assert_eq!(pool.state.lock().active, 0); + } + + #[test] + fn unrelated_wake_storm_never_renews_absolute_deadline() { + let deadline = 2_000_000_000; + // Each wake is allowed to report the full jiffy budget remaining. Only + // monotonic time, not that report or a wake count, bounds admission. + for now in (0..deadline).step_by(100_000) { + assert!(wait_slice_ms(deadline, now).unwrap() <= 10); + } + assert_eq!(wait_slice_ms(deadline, deadline), Err(EBUSY)); + assert_eq!(wait_slice_ms(deadline, deadline + 1), Err(EBUSY)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b78598cfbb760..713809907dd4b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -9,7 +9,12 @@ pub(crate) mod retirement; use crate::g17::{ channel::Rings, context::Context, - fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + fw::{ + channels::ControlRecord, + compute::{OperandState, COMPUTE_DESCRIPTOR_SIZE}, + queue::*, + variant::PointerLayout, + }, object::{Allocator, CpuMap, KernelObject}, }; use crate::{hw::t8140::dynamic, mmu}; @@ -113,17 +118,26 @@ struct Graph { records: KernelObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, + /// Host words of the pointer block in this firmware build. + pointers: PointerLayout, + /// Layout of the operand-state buffer for the GPU's cores. + operand_state: &'static OperandState, } impl Graph { - fn new(alloc: &Allocator<'_>, context: &Context) -> Result { + fn new( + alloc: &Allocator<'_>, + context: &Context, + pointers: PointerLayout, + operand_state: &'static OperandState, + ) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; let ring = DescriptorRing::new(alloc)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(0x14000, page, prot, cpu)?; + let operand = alloc.lower(operand_state.size, page, prot, cpu)?; let mut support = alloc.kernel(0xc000, page, prot, cpu)?; let support_va = support.gpu_va(); support.write(0, support_va + page)?; @@ -141,7 +155,7 @@ impl Graph { owner_pid: context.owner_pid(), policy: context.policy(), }); - view.pointers = PointerBlock::new(); + view.pointers = PointerBlock::new(&pointers)?; view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; @@ -160,6 +174,8 @@ impl Graph { records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, + pointers, + operand_state, }) } @@ -221,7 +237,7 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), + producer: self.graph.word(POINTERS + self.pointers.producer)?.load(Ordering::Relaxed), consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), }) @@ -258,7 +274,7 @@ impl Graph { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { let cursors = self.cursors()?; if !cursors.consumed(end, cursors.count)? { - return Err(EBUSY); + return Err(EAGAIN); } } Ok(()) @@ -276,7 +292,9 @@ impl Graph { self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); - self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); + self.graph + .word(POINTERS + self.pointers.producer)? + .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); Ok(()) @@ -389,11 +407,21 @@ struct Active { failure_deferred: bool, qos_pending: bool, } -struct Previous { - binding: Option, - context: Arc, +struct Previous> { + binding: Option, + context: C, publication: Option<(u64, u64)>, } +impl Previous { + /// The exact drain witness covers the replacement publication even when + /// its public fence failed. An unpublished replacement still needs rollback. + fn retire_witnessed(previous: &mut Option) { + if previous.as_ref().is_some_and(|p| p.publication.is_some()) { + drop(previous.take()); + } + } +} + /// Per-queue addresses of the bound owner, read under the publication preflight. struct BindingAddresses { preempt: u64, @@ -453,12 +481,21 @@ pub(crate) enum PoolPhase { Vacant, } +/// Builder ownership of one physical release transaction. The preparation +/// lease pins this registry until abort/commit; owner close may remove lookups. +#[derive(Clone, Copy)] +pub(crate) struct Growth { + qid: u8, + phase: PoolPhase, +} + pub(crate) struct Queue { owner: Option, /// Scheduler/work pages of an exited last owner whose context was dropped; /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + release_sent_ns: Option, growing: bool, growth_blocks: usize, /// A killed queue was recycled: its pool/ring must be vacated and re-armed @@ -471,6 +508,8 @@ pub(crate) struct Queue { /// `None` after an exited owner was detached; a new owner binds afresh. binding: Option, previous: Option, + /// This binding has never crossed the successor visibility boundary. + unpublished_binding: bool, kick: kick::Queue, pool: ComputePool, graph: Graph, @@ -509,9 +548,11 @@ impl Queue { backing: Backing, owner: u64, context: Arc, + pointers: PointerLayout, + operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context)?; + let mut graph = Graph::new(alloc, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -520,6 +561,7 @@ impl Queue { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + release_sent_ns: None, growing: false, growth_blocks: 0, needs_fresh_backing: false, @@ -527,6 +569,7 @@ impl Queue { free_list_generation: 0, binding: Some(binding), previous: None, + unpublished_binding: true, kick, pool, graph, @@ -566,6 +609,21 @@ impl Queue { pub(crate) fn context(&self) -> Option<&Arc> { self.binding.as_ref().map(|binding| &binding.context) } + /// The first command after an owner change away from `context` has been + /// published and not retired: the queue still runs that scheduler's state. + pub(crate) fn switching_from(&self, context: &Arc) -> bool { + self.previous.as_ref().is_some_and(|previous| { + previous.publication.is_some() && Arc::ptr_eq(&previous.context, context) + }) + } + /// The context these queue records were bound to before an owner change + /// whose first command is not published yet. + pub(crate) fn pending_switch_from(&self) -> Option<&Arc> { + self.previous + .as_ref() + .filter(|previous| previous.publication.is_none()) + .map(|previous| &previous.context) + } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) @@ -574,18 +632,16 @@ impl Queue { pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined } - pub(crate) fn room(&self) -> bool { - !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT - } /// Quarantines without a pending witness no longer describe live pipe order. pub(crate) fn timeout_head(&self) -> Option { if self.quarantined && !self.awaiting_witness { return None; } self.active.front().map(|active| crate::g17::timeout::Head { - vm: Arc::as_ptr(active.packet.completion.status()) as usize, + vm: active.packet.context.vm().id(), published: active.published, finished: active.observed.is_some(), + failed: self.quarantined, }) } @@ -626,6 +682,25 @@ impl Queue { !self.quarantined && self.retire_pending } + /// A release has no completion event: keep observing the control consumer + /// until its witness lets the idle physical queue be re-armed. + pub(crate) fn pool_release_pending(&self) -> bool { + matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) + } + + pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { + self.pool_release_pending() && crate::g17::timeout::background_stalled(self.release_sent_ns, now) + } + + /// A closed owner can still retain its last commands or their completion + /// stamps. Admission may wait for retirement, without treating that close + /// as permission to reuse any firmware-owned backing. + pub(crate) fn release_settling(&self) -> bool { + // release_owner keeps the old owner while its last commands retire. + // Waiting only once owner is None misses precisely that interval. + release_pending(self.released, self.quarantined, self.active.len, self.retire_pending) + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending @@ -780,9 +855,7 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = front - .observed_at - .is_some_and(|at| now_ns().saturating_sub(at) >= retirement::STAMP_GRACE_NS); + let record_aged = retirement::record_aged(front.observed_at, now_ns()); let timestamps = retirement::poll( &ticket, front.observed, @@ -829,6 +902,7 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; + self.released_at = now_ns(); } } Ok(None) | Err(EAGAIN) => { @@ -929,6 +1003,10 @@ impl Queue { self.retirement_proved = false; self.replay_retirement = false; self.spared_deferred = false; + // The stamp and fully consumed ring proved the replacement retired. + // Keeping its predecessor only on the fault path prevents recycling + // forever: neither another retirement nor an exit kill can release it. + Previous::retire_witnessed(&mut self.previous); if self.spared_quarantine { self.spared_quarantine = false; self.quarantine_error = None; @@ -937,15 +1015,9 @@ impl Queue { self.quarantined = false; self.retire_pending = false; self.retirement_ready = true; - if self - .previous - .as_ref() - .is_some_and(|p| p.publication.is_some()) - { - drop(self.previous.take()); - } if self.released { self.owner = None; + self.released_at = now_ns(); } } Ok(()) @@ -1027,6 +1099,102 @@ impl Queue { } } +/// Validate the complete unpublished item budget before changing descriptors, +/// work counts or queue producers. Leave one item empty across ring wrap. +fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { + if cursors.count != ITEM_RING_ENTRIES { return Err(EIO); } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < if entry_signal { 4 } else { 3 } { + let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); + return Err(EAGAIN); + } + Ok(()) +} + +/// Retained commands are capacity, not a failed VM. Nothing has been written +/// when this check runs, so the outer publisher can wait for retirement under +/// its original admission deadline, just as it does for a full firmware ring. +fn retained_capacity( + quarantined: bool, released: bool, active: usize, replays: usize, replay_front: bool, +) -> Result { + if quarantined || released { + return Err(EBUSY); + } + if active >= IN_FLIGHT || (replays != 0 && !replay_front) { + return Err(EAGAIN); + } + Ok(()) +} + +/// Logical close promises a future healthy handoff, not immediate reuse. +/// Exact retirement still owns the transition. +fn release_pending(released: bool, quarantined: bool, active: usize, retiring: bool) -> bool { + released && !quarantined && (active != 0 || retiring) +} + +#[kunit_tests(asahi_g17_compute_admission)] +mod admission_tests { + use super::*; + + #[test] + fn retained_ring_pressure_waits_without_discarding_work() { + let mut active = Fifo::new(); + for sequence in 0..IN_FLIGHT { + active.push(sequence).unwrap(); + } + assert_eq!(retained_capacity(false, false, active.len, 0, false), Err(EAGAIN)); + assert_eq!(active.front(), Some(&0)); + assert_eq!(active.len, IN_FLIGHT); + assert_eq!(active.pop(), Some(0)); + assert_eq!(retained_capacity(false, false, active.len, 0, false), Ok(())); + assert_eq!(active.front(), Some(&1)); + } + + #[test] + fn replay_front_and_terminal_ownership_remain_distinct() { + assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); + assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); + assert_eq!(retained_capacity(false, false, IN_FLIGHT, 1, true), Err(EAGAIN)); + for (quarantined, released) in [(true, false), (false, true), (true, true)] { + assert_eq!(retained_capacity(quarantined, released, 0, 1, true), Err(EBUSY)); + } + } + + #[test] + fn item_refusal_is_retryable_and_preserves_the_empty_slot() { + let n = ITEM_RING_ENTRIES; + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, false), Ok(())); + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, true), Err(EAGAIN)); + assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, false), Ok(())); + assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, true), Err(EAGAIN)); + assert_eq!(item_capacity(Cursors { producer: 10, consumer: 10, count: n }, true), Ok(())); + } + + #[test] + fn bad_item_cursors_are_not_backpressure() { + let n = ITEM_RING_ENTRIES; + assert_eq!(item_capacity(Cursors { producer: 0, consumer: 0, count: 0 }, false), Err(EIO)); + assert_eq!(item_capacity(Cursors { producer: n, consumer: 0, count: n }, false), Err(ERANGE)); + assert_eq!(item_capacity(Cursors { producer: 0, consumer: n, count: n }, true), Err(ERANGE)); + } + + #[test] + fn closed_owner_waits_for_its_last_command() { + assert!(release_pending(true, false, 1, false)); + assert!(release_pending(true, false, 0, true)); + assert!(!release_pending(true, false, 0, false)); + } + + #[test] + fn live_or_quarantined_owners_do_not_promise_capacity() { + assert!(!release_pending(false, false, 1, true)); + assert!(!release_pending(true, true, 1, true)); + } +} + impl Queue { pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { if self.owner != Some(owner) { @@ -1107,6 +1275,7 @@ impl Queue { } context.mark_published(); self.owner = Some(owner); + self.unpublished_binding = true; self.released = false; self.retirement_ready = false; self.retired_by_teardown = false; @@ -1134,6 +1303,13 @@ impl Queue { } self.retirement_ready = true; } + // Rebinding an already detached queue has no Previous. If its + // first command is rejected or cancelled before publication, the + // empty active FIFO is still the original idle witness. No future + // completion can turn retirement_ready back on for this owner. + if self.previous.is_none() { + self.retirement_ready = true; + } if self.retirement_ready { self.owner = None; self.released_at = now_ns(); @@ -1259,6 +1435,7 @@ impl Queue { ); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); + self.release_sent_ns = Some(crate::g17::now_ns()); fence(Ordering::SeqCst); Ok(()) } @@ -1283,7 +1460,7 @@ impl Queue { let pages = self.pool.take_pages(); let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; + self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1298,12 +1475,11 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { - return Err(EAGAIN); - } + if !self.can_rearm() { return Err(EAGAIN); } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; + self.release_sent_ns = None; self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } @@ -1314,11 +1490,7 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { - if self.growing { - if blocks < self.growth_blocks { return Err(EAGAIN); } - self.growth_blocks = blocks; - return Ok(false); - } + if self.growing { return Err(EAGAIN); } if self.quarantined || self.released || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() || (self.installed && !self.retirement_ready) @@ -1327,7 +1499,8 @@ impl Queue { let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - } else { self.pool_phase = PoolPhase::Vacant; } + self.release_sent_ns = Some(crate::g17::now_ns()); + } else { self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) @@ -1344,16 +1517,35 @@ impl Queue { return Err(EAGAIN); } } + if alias.is_none() { return Err(EINVAL); } + let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; - let old_alias = binding.pool_alias.take(); - let old_pages = self.pool.take_pages(); - self.pool.rearm(pages.take().ok_or(EINVAL)?)?; - init.initialize_compute_pool_blocks(self.pool.id(), self.pool.page_list_va()?, blocks)?; - binding.pool_alias = alias.take(); + // No ownership moves before every fallible descriptor/header operation. + // The old release is consumed; no queue can use this row until commit. + init.initialize_compute_pool_blocks(self.pool.id(), fresh.page_list_va(), blocks)?; + self.pool.replace_pages(pages)?; + let old_alias = core::mem::replace(&mut binding.pool_alias, alias.take()); self.pool_phase = PoolPhase::Populated; + self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { _alias: old_alias, _pages: old_pages, _descriptors: None, _ring: None }) + Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) + } + + pub(crate) fn growth_token(&self) -> Option { + self.growing.then_some(Growth { qid: self.qid(), phase: self.pool_phase }) + } + + /// Transfer an abandoned builder's release back to the event worker. Do not + /// revoke the published release or free backing here: vacancy still requires + /// its exact control-consumer witness. A different transaction is untouched. + pub(crate) fn abort_growth(&mut self, token: Growth) -> Result> { + if !self.growing || self.qid() != token.qid || self.pool_phase != token.phase { + return Ok(None); + } + self.growing = false; + self.growth_blocks = 0; + self.release_owner().map(Some) } /// Exact host identity remains valid even when several commands share an ioctl ID. @@ -1436,14 +1628,17 @@ impl Queue { } /// Firmware is halted and publication is closed while its target is copied. pub(crate) fn publish_recovery_target(&self) -> Result { + let Some(target) = self.graph.pointers.recovery_target else { + return Ok(()); + }; let producer = self .graph .graph - .word(POINTERS + PointerBlock::PRODUCER)? + .word(POINTERS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + PointerBlock::RECOVERY_TARGET)? + .word(POINTERS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1535,7 +1730,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(POINTERS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -1618,6 +1813,7 @@ impl Queue { || dependencies .iter() .any(|dependency| dependency.qid() != qid); + item_capacity(self.graph.cursors()?, entry_signal)?; let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -1640,6 +1836,7 @@ impl Queue { kick: kick_timestamp, preempt_va: addresses.preempt, operand_state_va: addresses.operand, + operand: self.graph.operand_state, usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), @@ -1726,16 +1923,6 @@ impl Queue { } let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; let cursors = self.graph.cursors()?; - if cursors.count != ITEM_RING_ENTRIES { - return Err(EIO); - } - if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { - return Err(ERANGE); - } - let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; - if free < if entry_signal { 4 } else { 3 } { - return Err(ENOSPC); - } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; if first { @@ -1743,6 +1930,7 @@ impl Queue { host.prepare_compute_shared()?; packet.mark_published(); visible = true; + self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -1779,6 +1967,7 @@ impl Queue { Err(error) })?; visible = true; + self.unpublished_binding = false; let qos = host.qos_publish( qos::Owner { qid, @@ -1898,11 +2087,8 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - if !self.room() - && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) - { - return Err(EBUSY); - } + retained_capacity(self.quarantined, self.released, self.active.len, + self.replays.len, replay_front)?; if !packet.context.is_current() || !self .binding @@ -1969,7 +2155,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(POINTERS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); @@ -1982,3 +2168,153 @@ impl Queue { self.terminal.take() } } + +impl Queue { + pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + write!(f, "compute_queue qid={} owned={} released={} reusable={} quarantined={} awaiting_witness={} previous={} exit={} growing={} pool={:?} active={} total={} ", + self.qid(), u8::from(self.owner.is_some()), u8::from(self.released), u8::from(self.reusable()), + u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), + u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; + match self.graph.cursors() { + Ok(cursors) if cursors.count != 0 => writeln!(f, "items_used={} items_total={}", + cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, cursors.count - 1), + _ => writeln!(f, "items_unavailable=1"), + }?; + writeln!(f, "compute_settling qid={} release={} pool_release={}", self.qid(), + u8::from(self.release_settling()), u8::from(self.pool_release_pending()))?; + if let Some(binding) = self.binding.as_ref() { + binding.context.write_capacity(f, "compute", usize::from(self.qid()))?; + } + if let Some(previous) = self.previous.as_ref() { + previous.context.write_capacity(f, "compute_previous", usize::from(self.qid()))?; + } + Ok(()) + } +} + +#[kunit_tests(asahi_g17_fault_ownership)] +mod fault_ownership_tests { + use super::*; + use core::sync::atomic::AtomicUsize; + + struct Lease(Arc); + impl Lease { + fn new(used: &Arc) -> Self { + used.fetch_add(1, Ordering::Relaxed); + Self(used.clone()) + } + } + impl Drop for Lease { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Relaxed); + } + } + + // This tests the production predecessor ownership operation without firmware + // storage. Runtime tests must separately establish its drain/kill witness. + fn faulting_clients(publication: Option<(u64, u64)>) { + let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); + for _ in 0..512 { + let mut previous = Some(Previous { + binding: Some(Lease::new(&used)), + context: Lease::new(&used), + publication, + }); + assert_eq!(used.load(Ordering::Relaxed), 2); + Previous::retire_witnessed(&mut previous); + assert_eq!(previous.is_none(), publication.is_some()); + if publication.is_none() { + assert_eq!(used.load(Ordering::Relaxed), 2); + // An unpublished replacement returns its original owner on close. + drop(previous.take()); + } + assert_eq!(used.load(Ordering::Relaxed), 0); + Previous::retire_witnessed(&mut previous); + assert_eq!(used.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn faulted_published_replacements_return_to_baseline() { + faulting_clients(Some((1, 7))); + } + + #[test] + fn unpublished_replacements_survive_until_rollback() { + faulting_clients(None); + } + + #[test] + fn same_vm_replacement_without_binding_returns_its_context() { + let used = Arc::new(AtomicUsize::new(0), GFP_KERNEL).unwrap(); + let mut previous = Some(Previous:: { + binding: None, + context: Lease::new(&used), + publication: Some((9, 3)), + }); + Previous::retire_witnessed(&mut previous); + assert_eq!(used.load(Ordering::Relaxed), 0); + } +} + +impl Queue { + /// Offer an open client's idle physical channel to another ready job. + /// Logical queue/context ownership is unchanged. The existing binding is + /// retained through the successor's exact replacement witness. + pub(crate) fn share_idle(&mut self) -> Result { + if !self.installed || self.owner.is_none() || !self.retirement_ready + || self.growing || self.quarantined || self.awaiting_witness + || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 + || self.previous.is_some() || self.pool_phase != PoolPhase::Populated + { + return Ok(false); + } + self.release_owner()?; + Ok(self.reusable()) + } +} + +impl Queue { + pub(crate) fn can_rearm(&self) -> bool { + self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + } +} + +impl Queue { + /// A failed first attempt has no future completion to undo its binding. + /// Return the replaced aliases to the caller for off-lock destruction. The + /// caller validates the current preparation epoch and cancels only a true + /// result's never-published QID; installed rings and identities stay intact. + pub(crate) fn rollback_unpublished( + &mut self, owner: u64, context: &Arc, + ) -> Result<(bool, Option)> { + if self.owner != Some(owner) || !self.unpublished_binding + || self.binding.as_ref().is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) + || self.growing || self.quarantined || self.awaiting_witness + || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 + || self.retire_pending || self.failure_status_pending + || self.pool_phase != PoolPhase::Populated + || self.previous.as_ref().is_some_and(|previous| previous.publication.is_some()) + { + return Ok((false, None)); + } + if !self.installed { return Ok((true, None)); } + let mut retired = None; + if let Some(previous) = self.previous.as_ref() { + self.graph.set_owner(&previous.context)?; + } + if let Some(mut previous) = self.previous.take() { + if let Some(binding) = previous.binding.take() { + retired = self.binding.replace(binding); + } else { + self.binding.as_mut().ok_or(EIO)?.context = previous.context; + } + } + self.unpublished_binding = false; + self.retirement_ready = true; + self.released = true; + self.owner = None; + self.released_at = now_ns(); + Ok((false, retired)) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 187fd27f8e78e..d7386e335e167 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -220,6 +220,9 @@ impl Queue { ))?; drop(self.active.pop()); } + // The acknowledged kill and command settlement replace the ordinary + // drain witness, including a failed first publication after rebinding. + super::Previous::retire_witnessed(&mut self.previous); self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; self.quarantined = true; self.awaiting_witness = false; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index b4d4bcef24421..e938ec566dbac 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -66,21 +66,37 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone. The firmware stores the stamp after the cache -/// maintenance that follows the record; it occasionally never stores it for a command, most -/// visibly while a render of the same VM is running. -pub(crate) const STAMP_GRACE_NS: u64 = 1_000_000; +/// before it retires on the record alone. +/// +/// The firmware stores the stamp after the cache maintenance that follows the record, so the +/// stamp, not the record, proves that the command's writes are complete and that its mappings +/// may be released. While other commands of the same submit ioctl are still running, the stamp +/// routinely lands several milliseconds after the record (on J700 under games, up to about 3 ms +/// after the 1 ms this used to wait). Only a stamp still missing well past that is treated as +/// never stored. The bound stays far below the scheduler timeout, so such a command cannot fail +/// its VM. +pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; + +// A command that waits out the fallback must still retire well inside the scheduler timeout. +const _: () = assert!(STAMP_FALLBACK_NS * 4 <= crate::g17::job::TIMEOUT_MS as u64 * 1_000_000); + +/// Whether a completion record latched at `observed_at` (host time, ns) has waited out +/// [`STAMP_FALLBACK_NS`] by `now`. +pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { + observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) +} static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); -/// Counts a command that retired on its record without its completion stamp; logs the first and -/// every 64th. +/// Counts a command that retired on its record because its completion stamp never arrived; +/// logs the first and every 64th. pub(crate) fn note_stampless(qid: u8, kick: u64) { let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; if count == 1 || count % 64 == 0 { pr_info!( - "G17: {} compute commands retired on their completion record without a completion stamp (latest: queue {} kick {:#x})\n", + "G17: {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", count, + STAMP_FALLBACK_NS / 1_000_000, qid, kick ); @@ -90,7 +106,12 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has -/// been present for [`STAMP_GRACE_NS`] (`record_aged`); in both cases its items must be consumed. +/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`); in both cases its items +/// must be consumed. +#[expect( + clippy::too_many_arguments, + reason = "Retirement validates the ticket against independent firmware observations." +)] pub(crate) fn poll( ticket: &Ticket, observed: Option, @@ -208,7 +229,8 @@ impl Batch { if descriptor == 0 || record.read(7)? == 0 { return Ok(()); } - let kick = (((control >> 8) & 0xffff_ffff_00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; + let kick = + (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; let Some(request) = requests .iter_mut() @@ -240,3 +262,143 @@ impl Batch { Ok(request.observed.take()) } } + +#[kunit_tests(asahi_g17_compute_stamp_wait)] +mod tests { + use super::*; + + const KICK: u64 = 0x12_3456_7801; + + fn ticket() -> Ticket { + Ticket { + owner: 1, + submission: 1, + context_id: 1, + qid: 28, + descriptor: 0x1000, + work_node: 0x2000, + kick: KICK, + ordinal: 1, + item_producer: 4, + item_count: 64, + epoch: 1, + } + } + + fn consumed() -> Cursors { + Cursors { + producer: 4, + consumer: 4, + count: 64, + } + } + + fn own(kick: u64) -> u32 { + (kick as u32).wrapping_shl(8) + } + + #[test] + fn stamp_covers_its_kick_and_earlier_ones() { + assert!(stamp_covers(own(KICK), KICK)); + assert!(stamp_covers(own(KICK + 1), KICK)); + assert!(!stamp_covers(own(KICK - 1), KICK)); + assert!(!stamp_covers(own(KICK) | 1, KICK)); + assert!(stamp_covers(0, 0xff_ffff)); + } + + #[test] + fn a_late_stamp_is_waited_for() { + // Under games on J700 the stamp lands up to about 4 ms after the record while the + // rest of the command's ioctl runs: the command keeps its mappings until then. + let record = Some(Observation { timestamps: [10, 20] }); + let at = 1_000_000; + assert!(!record_aged(Some(at), at + 4_000_000)); + assert!(!record_aged(Some(at), at + 10_000_000)); + let waiting = poll( + &ticket(), + record, + consumed(), + own(KICK - 1), + Recordless::NotYet, + 0, + None, + record_aged(Some(at), at + 10_000_000), + ); + assert_eq!(waiting, Ok(None)); + let stamped = poll( + &ticket(), + record, + consumed(), + own(KICK), + Recordless::NotYet, + 0, + None, + false, + ); + assert_eq!(stamped, Ok(Some([10, 20]))); + } + + #[test] + fn a_later_kick_stamp_retires_it() { + let record = Some(Observation { timestamps: [10, 20] }); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK + 3), + Recordless::NotYet, + 0, + None, + false, + ); + assert_eq!(result, Ok(Some([10, 20]))); + } + + #[test] + fn a_stamp_that_never_comes_retires_on_the_record_after_the_fallback() { + let record = Some(Observation { timestamps: [0x1111, 0x2222] }); + let at = 1_000_000; + assert!(record_aged(Some(at), at + STAMP_FALLBACK_NS)); + assert!(!record_aged(None, u64::MAX)); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK - 1), + Recordless::NotYet, + 0, + None, + record_aged(Some(at), at + STAMP_FALLBACK_NS), + ); + // The record's own timestamps are published, never synthesized ones. + assert_eq!(result, Ok(Some([0x1111, 0x2222]))); + } + + #[test] + fn an_aged_record_still_waits_for_consumed_items() { + let record = Some(Observation { timestamps: [10, 20] }); + let busy = Cursors { + producer: 4, + consumer: 2, + count: 64, + }; + let result = poll(&ticket(), record, busy, 0, Recordless::Allowed, 0, None, true); + assert_eq!(result, Ok(None)); + } + + #[test] + fn a_zero_duration_record_fails() { + let record = Some(Observation { timestamps: [5, 5] }); + let result = poll( + &ticket(), + record, + consumed(), + own(KICK), + Recordless::Allowed, + 0, + None, + true, + ); + assert_eq!(result, Err(ENODATA)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 569b5e289e7d8..6a9b15789339f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -56,10 +56,16 @@ pub(crate) struct Frontend { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, } impl Frontend { - pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { + pub(crate) fn new( + priority: u32, + usc_base: u64, + range: Range, + variant: &'static crate::g17::fw::Variant, + ) -> Result { if priority > 3 { return Err(EINVAL); } @@ -71,6 +77,7 @@ impl Frontend { allocation_failure, range, window, + variant, }) } } @@ -85,6 +92,7 @@ pub(crate) struct Queue { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], @@ -94,6 +102,8 @@ pub(crate) struct Queue { _slot: admission::QueueSlot, } +type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); + impl Queue { pub(crate) fn new( dev: &AsahiDrmDevice, @@ -107,6 +117,7 @@ impl Queue { allocation_failure, range, window, + variant, } = frontend; let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); Ok(Self { @@ -119,6 +130,7 @@ impl Queue { allocation_failure, range, window, + variant, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), @@ -128,7 +140,8 @@ impl Queue { } fn ensure_compute(&mut self, scs_pages: u16) -> Result { - // Admission is checked even when this queue already owns a scheduler. + // Validate every request, including an existing scheduler. Physical + // admission happens after its DRM dependencies, in backend publication. self.backend.ensure_compute(scs_pages)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( @@ -156,6 +169,10 @@ impl Queue { }); } + #[expect( + clippy::too_many_arguments, + reason = "Enqueue keeps validated command, ordering, and destination owners explicit." + )] fn enqueue( &mut self, id: u64, @@ -270,6 +287,13 @@ impl Queue { // either order and must each consume an input's error status before running. for sync in inputs { if let Some(fence) = &sync.fence { + // An input that already signalled without error orders nothing and has no status + // left to consume (like a completed frontier producer above). Keeping it would + // send the render through the scheduler thread instead of inline publication: + // most compositor and client renders carry such already-signalled buffer fences. + if job::fence_status(fence) > 0 { + continue; + } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } @@ -307,7 +331,7 @@ impl Queue { &mut self, bytes: &[u8], objects: Pin<&xarray::XArray>>, - ) -> Result<(KVec<(Validated, Order, Destinations)>, KVec)> { + ) -> Result { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); // All historical ordinals use the same entry counters; publication @@ -320,6 +344,7 @@ impl Queue { self.context.vm(), self.range.clone(), &self.window, + self.variant, &command.payload, )?; let engine = match &payload { @@ -384,7 +409,10 @@ impl crate::queue::Queue for Queue { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, guards) = self.prepare_submission(bytes, objects)?; + let (mut pending, guards) = self.prepare_submission(bytes, objects)?; + if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { + for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + } let mut guards = guards.into_iter(); let aggregate = outputs.publish(self.context.clone())?; for (command, order, destinations) in pending { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 99be455b48e40..fb6c77fbdf04f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -87,6 +87,8 @@ pub(crate) trait Host { fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; + /// The next publication number, shared with compute (timeout ordering). + fn next_publication(&mut self) -> Result; fn publish_outer_pair( &mut self, priority: u8, @@ -120,6 +122,8 @@ struct Prepared { mcache: Option, } struct Active { + /// Publication number, shared with compute (crate::g17::timeout). + published: u64, qos_completed: bool, flist_completed: bool, outer_published: bool, @@ -152,8 +156,11 @@ pub(crate) struct Pair { owner: Option, ordinal: u64, initial_packet: u64, + graph_started: bool, clusters: u32, + deflake: abi::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -168,14 +175,28 @@ pub(crate) struct Pair { previous: Option, displaced: Option, context_update: bool, - /// A retired, never-executed group awaiting republication on this pair. - replay: Option>, + /// Retired, never-executed groups awaiting republication on this pair, in + /// their original order. Nothing else is prepared on the pair before them. + replay: Tracker>, + /// When the oldest parked group was parked or the last one republished. + replay_since: Option, + /// Scheduler page of the context of the last published command: the + /// scheduler the firmware last ran these queues for. Zero for a fresh graph. + executed: u64, + /// An owner change in flight: the previous scheduler page and the ordinal of + /// the new owner's first command, until that command leaves the pair. + switch: Option<(u64, u64)>, } impl Pair { #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "The packet constructor spells out the paired render firmware inputs." + )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, + rings: &Arc, ids: [kick::Id; 2], slot: u8, owner: u64, @@ -186,7 +207,9 @@ impl Pair { pm_generation: u64, first: &Arc, clusters: u32, + deflake: abi::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> Result> { if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS @@ -211,10 +234,11 @@ impl Pair { if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters)?; + let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; + let (fragment, fragment_alias) = + kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -226,9 +250,10 @@ impl Pair { pm_generation, 0, word, + variant, )?; storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + kick_aliases: Some([ta_alias, fragment_alias]), memory, kicks: [tiling, fragment], free_list, @@ -239,8 +264,11 @@ impl Pair { owner: Some(owner), ordinal: 0, initial_packet: first.id, + graph_started: false, clusters, + deflake, descriptor_flags, + variant, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -251,8 +279,11 @@ impl Pair { terminal: None, previous: None, displaced: None, - replay: None, + replay: Tracker::new(), + replay_since: None, context_update: true, + executed: 0, + switch: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -280,20 +311,25 @@ impl Pair { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 || self.replay.is_some() + self.active.len() != 0 || self.replay.len() != 0 } /// A replay or an abandonment grace period needs the worker to poll. pub(crate) fn replay_polling(&self) -> bool { - self.replay.is_some() || self.active.iter().any(|a| a.abandoned_since.is_some()) + self.replay.len() != 0 || self.active.iter().any(|a| a.abandoned_since.is_some()) + } + pub(crate) fn replay_stalled(&self, now: u64) -> bool { + self.replay.len() != 0 && crate::g17::timeout::background_stalled(self.replay_since, now) } pub(crate) fn replay_front(&self) -> Option> { - self.replay.clone() + self.replay.front().cloned() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self.replay.as_ref().is_some_and(|front| Arc::ptr_eq(front, packet)) { + if !self.replay.front().is_some_and(|front| Arc::ptr_eq(front, packet)) { return Err(EIO); } - self.replay = None; + self.replay.pop(); + // Progress: the expiry restarts for the next parked group, if any. + self.replay_since = (self.replay.len() != 0).then(crate::g17::now_ns); Ok(()) } pub(crate) fn published_in_flight(&self) -> bool { @@ -315,6 +351,7 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() + || self.replay.front().is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { @@ -334,14 +371,17 @@ impl Pair { let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let deflake_layout = self.deflake; let owners = self.memory.owners(); self.memory .target .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; let target = self.memory.target.prepare(ordinal, &layout)?; let scene = self.memory.manager.prepare_scene(ordinal)?; - self.memory.generation(ordinal)?.prepare(ordinal, pass)?; + self.memory + .generation(ordinal)? + .prepare(ordinal, pass, deflake_layout.viewport)?; let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = self.memory.generation(ordinal)?.addresses(); let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { @@ -371,6 +411,7 @@ impl Pair { heapmeta_va: target.heap, tpc_va: self.memory.target.tpc_va(), tpc_size: layout.tpc_size as u64, + gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, fragment_status_va: fragment_status, @@ -402,11 +443,18 @@ impl Pair { stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], }, + programs: self.variant.programs, + deflake: deflake_layout, }; self.memory.graph.claim(ordinal)?; + // Without an MCache table kick-entry bit 58 stays clear. + let attachments = match self.variant.mcache { + true => attachments[1].as_slice(), + false => &[][..], + }; let result = self.memory.write_descriptors( &args, - attachments[1].as_slice(), + attachments, packet.context.id().try_into()?, ); let mcache = match result { @@ -503,9 +551,17 @@ impl Pair { }; use core::sync::atomic::{fence, Ordering}; let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; + self.graph_started = true; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let qids = self.qids(); + let kick_ids = self.kicks.each_ref().map(kick::Queue::id); + // Command preparation assigns context channel state before publishing + // producers. Pin identities before that assignment; warming never + // enters this path and can still give its unused identities back. + for id in kick_ids { + host.publish_qid(id)?; + } let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); let work = packet.completion.work_state()?; @@ -517,6 +573,13 @@ impl Pair { let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); self.memory .completion_links(ordinal, kicks, payload, !covered[1])?; + let leads = match self.variant.lead_commands { + true => { + let init_manager = !self.memory.manager.initialized(); + self.memory.write_leads(ordinal, kicks[0], init_manager)? + } + false => [false; 2], + }; let context = &packet.context; let ring = 1u8 << policy.priority(); let qos = context.qos_id(); @@ -619,6 +682,7 @@ impl Pair { abi::TaDescriptor::register_bindings( self.memory.graph.descriptor_client(0, ordinal), ray_query[0].scratch.enabled(), + self.variant.programs, )?, None, ); @@ -628,6 +692,7 @@ impl Pair { abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), ray_query[1].scratch.enabled(), + self.variant.programs, )?, mcache, ); @@ -649,6 +714,7 @@ impl Pair { configs[0], present[0], kicks[0], + leads[0], )?, publication::Lane::new( &memory.graph, @@ -658,6 +724,7 @@ impl Pair { configs[1], present[1], kicks[1], + leads[1], )?, ]; let windows = [lanes[0].window, lanes[1].window]; @@ -694,8 +761,10 @@ impl Pair { let next_inner = (next_ordinal as u32).wrapping_mul(2); let flist_submission = self.free_list.prepare_submission(2)?; let pm_submission = memory.manager.prepare_submission()?; + let published = host.next_publication()?; self.active .push(Active { + published, qos_completed: false, flist_completed: false, outer_published: false, @@ -730,9 +799,6 @@ impl Pair { host.install_pair(®istration)?; self.registered = Some(policy.priority()); } - for kick in &self.kicks { - host.publish_qid(kick.id())?; - } host.publish_outer_pair( policy.priority(), &fragment_slot, @@ -747,7 +813,12 @@ impl Pair { ) })(); let outer_next = match result { - Ok(next) => next, + Ok(next) => { + if leads[0] { + self.memory.manager.mark_initialized(); + } + next + } Err(error) => { self.quarantined = true; if !packet.completion.spared() && self.terminal.is_none() { @@ -781,6 +852,11 @@ impl Pair { if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } + let scheduler = packet.context.scheduler_va(); + if self.executed != 0 && self.executed != scheduler { + self.switch = Some((self.executed, ordinal)); + } + self.executed = scheduler; packet.context.mark_published(); self.announce_publication( host, @@ -924,6 +1000,22 @@ impl Pair { .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) }) } + /// The oldest published group still running, for cross-VM timeout attribution + /// (crate::g17::timeout). A quarantined pair's work holds the GPU until the firmware drains + /// it, so it stays visible to other VMs' watchdogs, as compute does, marked as failed. + pub(crate) fn timeout_head(&self) -> Option { + self.active + .iter() + .filter(|active| active.outer_published) + .find(|active| active.drained_result.is_none()) + .map(|active| crate::g17::timeout::Head { + vm: active.packet.context.vm().id(), + published: active.published, + finished: false, + failed: self.quarantined, + }) + } + pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { self.active .iter() @@ -934,7 +1026,7 @@ impl Pair { .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) } pub(crate) fn replay_contains(&self, packet: &Arc) -> bool { - self.replay.as_ref().is_some_and(|replay| Arc::ptr_eq(replay, packet)) + self.replay.iter().any(|replay| Arc::ptr_eq(replay, packet)) } pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { self.active.iter().find_map(|active| { @@ -955,7 +1047,7 @@ impl Pair { let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; - let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); @@ -1084,161 +1176,218 @@ impl Pair { let qos = self.qos_owners(); let pool = self.free_list.id(); let buffer = self.buffer_id(); - loop { - let Some(front) = self.active.front_mut() else { - return Ok(false); - }; - if !front.outer_published { - return Ok(false); - } - if !Arc::ptr_eq(&front.packet.context, &self.memory.context) - || !self.memory.owns(front.ticket.ordinal) - { + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) + || !self.memory.owns(front.ticket.ordinal) + { + return Err(EIO); + } + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = if let Some(observed) = observed { + if observed.ticket != front.ticket { return Err(EIO); } - let mut observer = Observer { - memory: &self.memory, - qids, - host, - }; - let result = if let Some(observed) = observed { - if observed.ticket != front.ticket { - return Err(EIO); - } - retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) - } else { - retirement::poll(&mut front.ticket, &mut observer) - }; - let timestamps = match result { - Ok(retirement::Poll::Complete(timestamps)) => timestamps, - Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer), - Ok(retirement::Poll::Retry) => return Ok(false), - Err(error) => { - self.quarantine(error, defer)?; - return Ok(false); - } - }; - let ordinal = front.ticket.ordinal; - let context_update = front.context_update; - self.note_retirement(ordinal, context_update); - let front = self.active.front_mut().ok_or(EIO)?; - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.qos_completed = true; - front.flist_completed = true; + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } else { + retirement::poll(&mut front.ticket, &mut observer) + }; + let timestamps = match result { + Ok(retirement::Poll::Complete(timestamps)) => timestamps, + Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer, true), + Ok(retirement::Poll::Retry) => return Ok(false), + Err(error) => { + self.quarantine(error, defer)?; + return Ok(false); } - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok(timestamps), - ))?; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - return Ok(true); + }; + let ordinal = front.ticket.ordinal; + let context_update = front.context_update; + self.note_retirement(ordinal, context_update); + let front = self.active.front_mut().ok_or(EIO)?; + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.qos_completed = true; + front.flist_completed = true; } + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok(timestamps), + ))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + Ok(true) } - /// The front group was retired and unlinked by firmware without executing - /// (job list empty, stamps cover it, no recovery, fragment end never - /// written). Nothing of this group reached its attachments, so the same - /// command is published again instead of failing the owner's VM. The - /// proof is the strict timeout witness, taken twice ABANDON_GRACE_NS apart, - /// and only for a lone group whose completion still accepts a replay. + /// Every retained group was retired and unlinked by firmware without + /// executing (job list empty, stamps cover it, no recovery, fragment end + /// never written): after another VM's recovery the firmware drops an + /// innocent pair's queued groups this way (seen on 25G83 and 26A428). + /// Nothing of them reached their attachments, so the same commands are + /// published again, in order, instead of failing their owners. Only the + /// whole set is replayed; a replayed prefix would run after a live + /// successor. `grace` asks for the strict witness twice ABANDON_GRACE_NS + /// apart (worker polling); the render timeout, long after publication, + /// takes it once as its retirement does. A single sample can still + /// straddle a firmware update of the fragment end record; the cost is + /// that a group which did execute runs a second time (blending twice). fn replay_abandoned( &mut self, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + grace: bool, ) -> Result { use retirement::Memory; - if self.quarantined || self.active.len() != 1 || self.replay.is_some() { + if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { return Ok(false); } - let front = self.active.front().ok_or(EIO)?; - if !front.outer_published || front.failed { - return Ok(false); + use crate::g17::replay::{self, Decision, Group}; + let qids = self.qids(); + let memory = &self.memory; + let mut failure = None; + let groups = self.active.iter().map(|active| { + let innocent = + active.outer_published && !active.failed && !active.packet.completion.guilty(); + let blamed = active.packet.completion.blamed(); + let exhausted = + active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; + // decide() stops at the first group that is not abandoned, an error included. + let abandoned = innocent + && !blamed + && (|| -> Result { + let mut observer = Observer { + memory, + qids, + host: &mut *host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + // A torn sample is no proof; a later poll or the timeout looks again. + Ok(matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, false), + Ok(true) + )) + })() + .unwrap_or_else(|error: Error| { + failure = Some(error); + false + }); + Group { + innocent, + blamed, + abandoned, + exhausted, + } + }); + let decision = replay::decide(groups); + if let Some(error) = failure { + return Err(error); } - let mut observer = Observer { - memory: &self.memory, - qids: self.qids(), - host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(front.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - let abandoned = - retirement::abandoned(&front.ticket, snapshot, stamps, state, false)?; let now = crate::g17::now_ns(); let front = self.active.front_mut().ok_or(EIO)?; - if !abandoned { - front.abandoned_since = None; - return Ok(false); + match decision { + Decision::Refuse => return Ok(false), + Decision::Wait => { + front.abandoned_since = None; + return Ok(false); + } + Decision::Replay => {} } - let Some(since) = front.abandoned_since else { - front.abandoned_since = Some(now); - return Ok(false); - }; - if now.saturating_sub(since) < ABANDON_GRACE_NS { - return Ok(false); + if grace { + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(false); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(false); + } } - if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { - pr_info!( - "render slot {} retired without execution (pass {:?}); replaying\n", + // decide() lets only the front be exhausted. An exhausted front fails, and its + // successors, which consume its output, fail with it rather than run without it. + let mut failing = self.active.front().is_some_and(|front| { + front.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS + }); + if failing { + pr_warn!( + "G17: render slot {}: {} groups retired without execution; replay limit reached, failing them\n", + self.slot, + self.active.len() + ); + } else if !grace || crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + pr_warn!( + "G17: render slot {}: {} groups retired without execution; replaying\n", self.slot, - snapshot.pass + self.active.len() ); } let qos = self.qos_owners(); let buffer = self.buffer_id(); let pool = self.free_list.id(); - let front = self.active.front_mut().ok_or(EIO)?; - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.flist_completed = true; - front.qos_completed = true; - } - let ordinal = front.ticket.ordinal; - let packet = front.packet.clone(); - if !packet.completion.begin_replay() { - if !packet.completion.spared() { - packet.completion.status().record(EIO); - if self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), EIO)); + // Every group parked below shares this expiry (an early return keeps it). + self.replay_since = Some(now); + while let Some(front) = self.active.front_mut() { + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let packet = front.packet.clone(); + failing = failing || !packet.completion.begin_replay(); + if failing { + if !packet.completion.spared() { + packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), EIO)); + } } + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + continue; } - defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; - return Ok(true); + packet.reset_publication_for_replay(); + self.replay.push(packet).map_err(|_| EIO)?; } - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - packet.reset_publication_for_replay(); - self.replay = Some(packet); Ok(true) } /// The pending replay cannot proceed (closed owner, cancellation or a - /// publication refusal). ECANCELED leaves the VM usable; any other error - /// is the ordinary device-loss outcome the replay tried to avoid. + /// publication refusal). Its successors cannot run without it and fail with + /// it. ECANCELED leaves the VM usable; any other error is the ordinary + /// device-loss outcome the replay tried to avoid. pub(crate) fn fail_replay( &mut self, error: Error, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - let Some(packet) = self.replay.take() else { - return Ok(()); - }; - defer(Deferred::Retired(packet.completion.clone(), Err(error))) + self.replay_since = None; + while let Some(packet) = self.replay.pop() { + defer(Deferred::Retired(packet.completion.clone(), Err(error)))?; + } + Ok(()) } /// A closed preparation epoch invalidates only this unpublished descriptor. @@ -1357,22 +1506,51 @@ impl Pair { if !self.active.iter().any(|a| Arc::ptr_eq(&a.packet, packet)) { return Ok(false); } - if !self.quarantined && self.active.len() == 1 { - let active = self.active.front().ok_or(EIO)?; - let mut observer = Observer { - memory: &self.memory, - qids: self.qids(), - host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(active.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - if active.outer_published - && retirement::abandoned(&active.ticket, snapshot, stamps, state, false)? - { + // Groups the firmware dropped unexecuted are published again, with their successors. + if self.replay_abandoned(host, defer, false)? { + return Ok(true); + } + // Fallback when a live or exhausted group prevents an ordered replay: a spared front group + // the firmware retired without executing during another VM's recovery is retired on its + // own even with successors queued, so the pair is not quarantined into a sticky EIO. It + // fails with ECANCELED, which spared work never reports as device loss; consecutive + // abandoned spared successors are retired the same way. + let front_spared = self.active.front().is_some_and(|a| { + Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared() + }); + if !self.quarantined && (self.active.len() == 1 || front_spared) { + let mut retired = false; + loop { + let Some(active) = self.active.front() else { + break; + }; + let spared = active.packet.completion.spared(); + if retired && !spared { + break; + } + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + // A torn sample is no proof, as in replay_abandoned(): the group stays retained + // (quarantined with the pair if nothing was retired) instead of the error + // stopping the runtime for every client. + if !active.outer_published + || !matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, false), + Ok(true) + ) + { + break; + } let qos = self.qos_owners(); let buffer = self.buffer_id(); + let left = self.active.len() - 1; let active = self.active.front_mut().ok_or(EIO)?; if !active.flist_completed { host.complete_render( @@ -1384,19 +1562,30 @@ impl Pair { active.flist_completed = true; active.qos_completed = true; } - if !active.failed && !active.packet.completion.spared() { + if !active.failed && !spared { active.packet.completion.status().record(EIO); if self.terminal.is_none() { self.terminal = Some((active.packet.completion.status().clone(), error)); } } + pr_warn!( + "G17: render retired unexecuted after a recovery (ordinal {}, {} groups left)\n", + active.ticket.ordinal, + left + ); defer(Deferred::Retired( active.packet.completion.clone(), - Err(EIO), + Err(if spared { ECANCELED } else { EIO }), ))?; let ordinal = active.ticket.ordinal; self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; + retired = true; + if !spared { + break; + } + } + if retired { return Ok(true); } } @@ -1424,7 +1613,8 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; } - if let Some(packet) = self.replay.take() { + self.replay_since = None; + while let Some(packet) = self.replay.pop() { defer(Deferred::Drained(packet.completion.clone(), Err(EIO)))?; } if let Some(previous) = self.previous.as_mut() { @@ -1578,6 +1768,18 @@ impl Pair { } } } + /// The scheduler page these queues last ran a command for, when the next + /// publication for `context` changes the queues' owner. + pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { + (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) + } + /// The previous scheduler page of an owner change whose first command has + /// not left the pair yet. + pub(crate) fn switch_in_flight(&self) -> Option { + let (from, first) = self.switch?; + (self.replay.len() != 0 || self.active.iter().any(|active| active.ticket.ordinal == first)) + .then_some(from) + } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { return self.displaced.take(); @@ -1621,6 +1823,10 @@ impl Pair { self.owner = None; } #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Binding keeps the render owners and first packet explicit." + )] pub(crate) fn prepare_bind( &mut self, alloc: &Allocator<'_>, @@ -1654,7 +1860,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, @@ -1670,6 +1876,7 @@ impl Pair { pm_generation, self.ordinal, first.completion.work_state()?.word(), + self.variant, )?; (Some(memory), Some(persistent), Some(aliases)) }; @@ -1736,6 +1943,7 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; + if foreign { self.graph_started = false; } self.context_update = true; if foreign { self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 85319585b3d13..80ee0b871cc82 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,6 +31,9 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; +/// The lead command of a stage (InitBM before tiling, the barrier before fragment), after the +/// stage's configuration record in its slot ([`crate::g17::fw::Variant::lead_commands`]). +pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); @@ -94,6 +97,8 @@ pub(super) struct Graph { pub(super) config: PooledObject, pub(super) support: KernelObject, pub(super) owners: [Option; RECORDS], + /// Host words of the pointer blocks in this firmware build. + pub(super) pointers: fw::variant::PointerLayout, } impl Graph { pub(super) fn new( @@ -101,6 +106,7 @@ impl Graph { pool: &Arc, context: &Context, qids: [u8; 2], + pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; let descriptors = pool.allocate( @@ -154,7 +160,7 @@ impl Graph { }), )?; queues.initialize::(POINTERS[stage], |view| { - *view = queue::PointerBlock::new(); + *view = queue::PointerBlock::new(&pointers)?; Ok(()) })?; } @@ -166,6 +172,7 @@ impl Graph { config, support, owners: [None; RECORDS], + pointers, }) } pub(super) fn offset(stage: usize, ordinal: u64) -> usize { @@ -188,11 +195,13 @@ impl Graph { self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 } pub(super) fn publish_recovery_targets(&self) -> Result { + let Some(target) = self.pointers.recovery_target else { + return Ok(()); + }; for at in POINTERS { - let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? + let producer = self.queues.word(at + self.pointers.producer)? .load(Ordering::Relaxed); - self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? - .store(producer, Ordering::Relaxed); + self.queues.word(at + target)?.store(producer, Ordering::Relaxed); } Ok(()) } @@ -208,7 +217,7 @@ impl Graph { values[stage] = Cursors { done: self.queues.word(at)?.load(Ordering::Relaxed), read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), - write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), + write: self.queues.word(at + self.pointers.producer)?.load(Ordering::Relaxed), }; } Ok(values) @@ -310,24 +319,36 @@ impl Generation { ) .map_err(Into::into) } - pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { + /// With `viewport` false the deflake buffer has no room for the viewport words + /// ([`abi::Deflake::viewport`]). + pub(super) fn prepare( + &mut self, + ordinal: u64, + pass: &abi::RenderPass, + viewport: bool, + ) -> Result { if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } - let viewport = storage::Viewport::new(pass.width, pass.height)?; + let viewport = match viewport { + true => Some(storage::Viewport::new(pass.width, pass.height)?), + false => None, + }; self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] .fill(0); - let destination = self - .state - .pointer(storage::DEFLAKE + 0x900, size_of::())? - .cast::(); - if destination.align_offset(align_of::()) != 0 { - return Err(EINVAL); + if let Some(viewport) = viewport { + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; } - // SAFETY: The complete typed integer view is in the retained CPU mapping - // and naturally aligned. This command generation is exclusively owned - // and unpublished; no prior command can access it until reassigned. - unsafe { viewport.apply(&mut *destination) }; self.fragment.bytes_mut().fill(0); self.ordinal = Some(ordinal); Ok(()) @@ -365,6 +386,10 @@ pub(super) struct Memory { } impl Memory { #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Render memory construction keeps each allocation and firmware input explicit." + )] pub(super) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -376,10 +401,11 @@ impl Memory { pm_generation: u64, ordinal: u64, word: u32, + variant: &'static fw::Variant, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); - let mut graph = Graph::new(alloc, pool, &context, qids)?; + let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; let state = Generation::allocate_state(alloc, vm)?; @@ -505,6 +531,9 @@ const _: () = { assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); + assert!(size_of::() <= LEAD); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; impl Memory { @@ -598,6 +627,52 @@ impl Memory { } Ok(()) } + + /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the + /// tiling descriptor when `init_manager`, and the fragment barrier. Returns which stages + /// have one. + pub(super) fn write_leads( + &mut self, + ordinal: u64, + tiling_kick: fw::kick::KickTimestamp, + init_manager: bool, + ) -> Result<[bool; 2]> { + if !self.owns(ordinal) { + return Err(EIO); + } + let ta = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + // SAFETY: Both views were checked during preparation and the ordinal still owns them; + // only integer fields are read. + let (ta, fragment_id) = unsafe { (&*ta, (*fragment).descriptor.current_id) }; + let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; + let barrier = queue::FragmentBarrier::new( + ta.stamp_va, + tiling_kick, + ta.qid, + ta.stamp_value, + fragment_id, + ); + self.graph.config.write(CONFIG[1] + slot, barrier)?; + if init_manager { + let init = queue::InitBufferManager::new( + ta.context_id, + ta.pb_slot, + ta.pb_va, + ta.stamp_value, + ); + self.graph.config.write(CONFIG[0] + slot, init)?; + } + Ok([init_manager, true]) + } } impl Memory { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 39941de815e07..209ce0b8176ec 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -25,12 +25,19 @@ pub(crate) struct ParkedPair { slot: u8, ordinal: u64, clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], recovery_generation: u64, registered: Option, outer_started: u8, previous: Option, + /// Scheduler page these queues last ran a command for. A pair parked + /// without a consumed release (an unused warm pair taken for another + /// owner) keeps it, so its next owner change still waits while that + /// scheduler is released. + executed: u64, } /// All new graph ownership stays separate from the installed kick storage until commit. @@ -48,11 +55,20 @@ pub(crate) struct Prepared { } impl Pair { + /// The current graph never entered command publication. Physical identities + /// may have older users; parking retains their rings and retirement owners. + pub(crate) fn can_yield_unused(&self) -> Result { + Ok(!self.graph_started && self.can_park_backing(self.context())?) + } + /// The registry also checks the shared free-list release and scheduler-release witness. /// Drain displaced owners before this preflight; they have their own logical releases. pub(crate) fn can_park(&self, context: &Arc) -> Result { + Ok(self.owner.is_none() && self.can_park_backing(context)?) + } + + fn can_park_backing(&self, context: &Arc) -> Result { Ok(Arc::ptr_eq(context, self.context()) - && self.owner.is_none() && self .previous .as_ref() @@ -76,12 +92,15 @@ impl Pair { slot: self.slot, ordinal: self.ordinal, clusters: self.clusters, + deflake: self.deflake, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, outer_started: self.outer_started, previous: self.previous, + executed: self.executed, }; let retired = RetiredBinding { memory: Some(self.memory), @@ -96,11 +115,22 @@ impl Pair { } impl ParkedPair { + /// The release of `scheduler` has been consumed: a later owner change no + /// longer has to wait for it. + pub(crate) fn release_consumed(&mut self, scheduler: u64) { + if self.executed == scheduler { + self.executed = 0; + } + } pub(crate) fn qids(&self) -> [u8; 2] { self.ids.map(kick::Id::qid) } #[inline(never)] + #[expect( + clippy::too_many_arguments, + reason = "Preparation forwards the render binding inputs without changing their ownership." + )] pub(crate) fn prepare( &self, alloc: &Allocator<'_>, @@ -125,7 +155,7 @@ impl ParkedPair { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, @@ -141,6 +171,7 @@ impl ParkedPair { pm_generation, self.ordinal, word, + self.variant, )?; Ok(Prepared { memory: Some(memory), @@ -190,6 +221,7 @@ impl ParkedPair { return Err(EIO); }; storage.write_init(kernel::try_init!(Pair { + graph_started: false, kick_aliases: None, memory, kicks, @@ -202,7 +234,9 @@ impl ParkedPair { ordinal: self.ordinal, initial_packet: next.initial_packet, clusters: self.clusters, + deflake: self.deflake, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, configs: [None; 2], cursors: [0; 2], @@ -213,8 +247,11 @@ impl ParkedPair { terminal: None, previous: self.previous.take(), displaced: None, - replay: None, + replay: Tracker::new(), + replay_since: None, context_update: true, + executed: self.executed, + switch: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 0fc9c1d9a15dc..9f4cb05b6bede 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -28,9 +28,9 @@ pub(super) fn required(previous: Option, config: QueueConfig) -> bo } pub(super) struct Lane<'a> { - items: [&'a AtomicU64; 3], + items: [&'a AtomicU64; 4], producer: &'a AtomicU32, - values: [u64; 3], + values: [u64; 4], config: QueueConfig, config_destination: *mut u64, config_present: bool, @@ -49,8 +49,9 @@ impl<'a> Lane<'a> { config: QueueConfig, present: bool, timestamp: KickTimestamp, + lead: bool, ) -> Result { - let count = 2 + u32::from(present); + let count = 2 + u32::from(present) + u32::from(lead); let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -72,21 +73,28 @@ impl<'a> Lane<'a> { graph.queues.dword(index(0))?, graph.queues.dword(index(1))?, graph.queues.dword(index(2))?, + graph.queues.dword(index(3))?, ]; let announce_va = graph.queues.gpu_va() + announce_offset as u64; let descriptor = graph.descriptor_va(stage, ordinal); - let values = if present { - [ - descriptor, - graph.config.gpu_va() + config_offset as u64, - announce_va, - ] - } else { - [descriptor, announce_va, 0] + // Item order: [lead command], descriptor, [configuration], announcement. + let mut values = [0; 4]; + let mut n = 0; + let mut push = |value: u64| { + values[n] = value; + n += 1; }; + if lead { + push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); + } + push(descriptor); + if present { + push(graph.config.gpu_va() + config_offset as u64); + } + push(announce_va); Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, + producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, @@ -116,7 +124,7 @@ impl<'a> Lane<'a> { if !self.config_present { return; } - let source = (&self.config as *const QueueConfig).cast::(); + let source = core::ptr::from_ref(&self.config).cast::(); for index in 0..size_of::() / 8 { // SAFETY: QueueConfig is packed integer storage with no padding; // every source qword is read unaligned. The destination is a diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index d0a2d566bdc8d..24aeed6bcd0ec 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -203,8 +203,12 @@ pub(crate) fn poll_observed( memory: &mut impl Memory, snapshot: Snapshot, ) -> Result { - let observation = observe(ticket, snapshot)?; - complete_poll(ticket, memory, snapshot, observation) + match observe(ticket, snapshot) { + Ok(observation) => complete_poll(ticket, memory, snapshot, observation), + // The borrowed snapshot can straddle a firmware cursor update like any other sample: + // take the ordinary poll's fresh snapshot and reread instead of failing the client. + Err(_) => poll(ticket, memory), + } } fn complete_poll( diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs index 4db471f254ec6..da90488bfedec 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -36,10 +36,11 @@ impl Pair { } *block = pointer.cast(); } + let layout = self.memory.graph.pointers; for block in blocks { // SAFETY: Both complete pointer blocks were checked above. This fresh // graph is unpublished; only the live atomic cursors are accessed. - unsafe { &*block }.rebase(0); + unsafe { &*block }.rebase(&layout, 0)?; } fence(Ordering::SeqCst); self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index ba368341d84d2..89f8d33bdb80c 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -9,11 +9,12 @@ mod attribution; mod slot; +mod safety; pub(crate) use attribution::Sources; use attribution::Attribution; -pub(super) fn slot_key(slot: usize) -> Option { - slot::key(slot).ok() +pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { + slot::key(slot, slot_keys?).ok() } use super::status::VmStatus; @@ -99,6 +100,8 @@ struct Request { pub(super) struct State { accepted: u64, pending: Option, + timeouts: safety::Storm, + restarts: safety::Storm, } /// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. @@ -112,8 +115,12 @@ impl State { Self { accepted: 0, pending: None, + timeouts: safety::Storm::new(), + restarts: safety::Storm::new(), } } + pub(super) fn timeout_limit(&mut self, now: u64) -> bool { self.timeouts.note(now) } + pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted } @@ -220,9 +227,17 @@ pub(super) fn service( } host.recovery_mut().pending = None; - let cancelled = classify(host, entries, request.blamed, &mut sources); + let (cancelled, blame) = classify(host, entries, request.blamed, &mut sources); if let Some(qid) = request.blamed { - host.settle_blamed_render(qid, EIO); + pr_warn!( + "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", + sources.reason, + qid, + if blame { "failing its oldest render" } else { "attribution cleared it; spared" } + ); + if blame { + host.settle_blamed_render(qid, EIO); + } } if cancelled != 0 { host.settle_cancelled_renders(cancelled); @@ -235,7 +250,10 @@ pub(super) fn service( /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> u128 { +/// Also returns whether the firmware's blamed QID is still guilty after attribution: a blamed +/// render that had not started while others had (or with no culprit at all) is innocent, and its +/// work is spared like that of every other innocent VM instead of failing with EIO. +fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> (u128, bool) { let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); let cancelled = entries.cancelled; let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); @@ -245,8 +263,9 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou Attribution::NoCulprit => guilty_qids = 0, } let no_culprit = attribution == Attribution::NoCulprit; + let blame = blamed.is_some_and(|qid| guilty_qids & (1u128 << qid) != 0); if guilty_qids == 0 && !no_culprit { - return 0; + return (0, blame); } let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); let mut count = 0; @@ -282,7 +301,7 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou } } if unresolved || (count == 0 && collateral == 0 && !no_culprit) { - return 0; + return (0, blame); } let spared = queues.classify_work(&|status| { guilty[..count] @@ -295,5 +314,5 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou }) .unwrap_or(Class::Spared) }); - cancelled & !(guilty_qids & !collateral) & spared + (cancelled & !(guilty_qids & !collateral) & spared, blame) } diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs new file mode 100644 index 0000000000000..c30e45e22579a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fail closed on repeated watchdog/restart traffic, even across client loss. + +const WINDOW_NS: u64 = 30_000_000_000; +const LIMIT: u8 = 8; + +pub(super) struct Storm { + start: Option, + count: u8, + stopped: bool, +} +impl Storm { + pub(super) const fn new() -> Self { + Self { start: None, count: 0, stopped: false } + } + pub(super) fn note(&mut self, now: u64) -> bool { + if self.stopped { return true; } + if self.start.is_none_or(|start| now.saturating_sub(start) >= WINDOW_NS) { + self.start = Some(now); + self.count = 0; + } + self.count += 1; + self.stopped = self.count >= LIMIT; + self.stopped + } +} + +#[kernel::macros::kunit_tests(asahi_g17_timeout_recovery_safety)] +mod tests { + use super::*; + use crate::g17::{preparation::Gate, recovery::State}; + use kernel::prelude::*; + + #[test] + fn consecutive_timeouts_during_recovery_are_bounded_and_lock_ordered() { + let gate = Gate::new().unwrap(); + let lease = gate.try_enter().unwrap(); + let state = KBox::pin_init(kernel::new_mutex!(State::new()), GFP_KERNEL).unwrap(); + gate.close(); + for n in 0..64 { + let stopped = { + let mut state = state.lock(); + state.request(0, -1).unwrap(); + assert!(state.pending()); + let stopped = state.timeout_limit(n * 2_000_000_000); + if stopped { gate.fail(); } // production state -> gate order + stopped + }; + assert_eq!(stopped, n >= u64::from(LIMIT - 1)); + } + // Recovery/stop must release state before joining the retained builder. + assert!(matches!(gate.wait_drained_until(crate::g17::now_ns() + 2_000_000), + Err(error) if error == ETIMEDOUT)); + assert!(!gate.reopen()); // failure never gives back an owned reservation + drop(lease); + gate.wait_drained_until(crate::g17::now_ns() + 100_000_000).unwrap(); + assert!(!gate.reopen()); + } + #[test] + fn successful_handshakes_do_not_reset_a_restart_storm() { + let mut state = State::new(); + for n in 0..64 { + assert_eq!(state.restart_limit(n), n >= u64::from(LIMIT - 1)); + } + assert!(state.restart_limit(u64::MAX)); + } + #[test] + fn separated_faults_do_not_accumulate_and_latched_loss_cannot_wrap() { + let mut storm = Storm::new(); + for n in 0..64 { assert!(!storm.note(n * WINDOW_NS)); } + for _ in 1..LIMIT { storm.note(64 * WINDOW_NS); } + assert!(storm.note(64 * WINDOW_NS)); + for _ in 0..1024 { assert!(storm.note(u64::MAX)); } + } + #[test] + fn replay_dependency_checks_never_wait_for_event_service() { + use crate::g17::dependency::replay_status; + assert_eq!(replay_status(0), Err(EBUSY)); + assert_eq!(replay_status(1), Ok(())); + assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED)); + assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs index 307534708e922..bd9d4c96be14f 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/slot.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -2,7 +2,7 @@ //! Bounded read-only slot identity sampling while firmware is halted. -use crate::{hw::t8140, mmu}; +use crate::mmu; use core::{ ffi::c_void, ptr::NonNull, @@ -19,11 +19,11 @@ impl Drop for Mapping { } } -pub(super) fn key(slot: usize) -> Result { +pub(super) fn key(slot: usize, slot_keys: u64) -> Result { if slot >= super::attribution::SLOTS { return Err(EINVAL); } - let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; + let physical = slot_keys + slot as u64 * size_of::() as u64; let offset = physical as usize & (mmu::UAT_PGSZ - 1); // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here // for mapping. Only the selected aligned eight-byte word is read below. @@ -31,7 +31,7 @@ pub(super) fn key(slot: usize) -> Result { bindings::memremap( physical - offset as u64, mmu::UAT_PGSZ, - bindings::MEMREMAP_WB as _, + bindings::MEMREMAP_WB as usize, ) }; let mapping = Mapping(NonNull::new(raw).ok_or(ENOMEM)?); diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs new file mode 100644 index 0000000000000..94daa5ec0dfeb --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/replay.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Whether a pair's retained render groups, retired by firmware without executing, are +//! published again. Only the whole set is replayed, so no replayed group runs after a live +//! successor, and work a recovery blamed is never run again. + +/// One retained group, front first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Group { + /// Published, not failed and not classified guilty by a recovery. + pub(crate) innocent: bool, + /// A recovery blamed it, whatever class it kept (an earlier recovery may have spared it): + /// it may have caused the fault. + pub(crate) blamed: bool, + /// Carries the strict abandonment witness. + pub(crate) abandoned: bool, + /// Already republished the maximum number of times. + pub(crate) exhausted: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Decision { + Replay, + /// A group may still execute: look again later. + Wait, + /// Never replay this set; the render timeout settles it. + Refuse, +} + +/// Groups are read in order and the first one that decides stops the walk, so a caller can +/// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its +/// successors, which consume its output, with it. An exhausted successor would break the order. +pub(crate) fn decide(groups: impl Iterator) -> Decision { + let mut front_exhausted = None; + let mut successor_exhausted = false; + for group in groups { + if !group.innocent || group.blamed { + return Decision::Refuse; + } + if !group.abandoned { + return Decision::Wait; + } + match front_exhausted { + None => front_exhausted = Some(group.exhausted), + Some(_) => successor_exhausted |= group.exhausted, + } + } + match front_exhausted { + Some(true) => Decision::Replay, + Some(false) if !successor_exhausted => Decision::Replay, + _ => Decision::Refuse, + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_render_replay))] +mod tests { + use super::*; + fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { + Group { innocent, blamed: false, abandoned, exhausted } + } + const OK: Group = Group { innocent: true, blamed: false, abandoned: true, exhausted: false }; + const BLAMED: Group = Group { innocent: true, blamed: true, abandoned: true, exhausted: false }; + #[test] + fn every_group_abandoned_and_innocent_is_replayed() { + assert_eq!(decide([OK].into_iter()), Decision::Replay); + assert_eq!(decide([OK, OK, OK].into_iter()), Decision::Replay); + } + #[test] + fn nothing_retained_is_never_replayed() { + assert_eq!(decide([].into_iter()), Decision::Refuse); + } + #[test] + fn guilty_work_is_never_replayed() { + assert_eq!(decide([group(false, true, false)].into_iter()), Decision::Refuse); + assert_eq!(decide([OK, group(false, true, false)].into_iter()), Decision::Refuse); + } + #[test] + fn blamed_work_is_never_replayed_even_when_spared() { + // An earlier recovery spared it, so it still counts as innocent; the blame refuses it. + assert_eq!(decide([BLAMED].into_iter()), Decision::Refuse); + assert_eq!(decide([BLAMED, OK].into_iter()), Decision::Refuse); + assert_eq!(decide([OK, BLAMED].into_iter()), Decision::Refuse); + // Also once the replay limit is reached, where an innocent front alone is replayed. + let exhausted = Group { exhausted: true, ..BLAMED }; + assert_eq!(decide([exhausted].into_iter()), Decision::Refuse); + } + #[test] + fn a_live_group_waits_and_stops_the_walk() { + assert_eq!(decide([OK, group(true, false, false)].into_iter()), Decision::Wait); + // Nothing after the live group is looked at, guilty or not. + assert_eq!( + decide([group(true, false, false), group(false, true, false)].into_iter()), + Decision::Wait + ); + let mut seen = 0; + let groups = [group(true, false, false), OK, OK].into_iter().inspect(|_| seen += 1); + assert_eq!(decide(groups), Decision::Wait); + assert_eq!(seen, 1); + } + #[test] + fn an_exhausted_successor_refuses_an_exhausted_front_replays() { + assert_eq!(decide([OK, group(true, true, true)].into_iter()), Decision::Refuse); + assert_eq!(decide([group(true, true, true), OK].into_iter()), Decision::Replay); + assert_eq!( + decide([group(true, true, true), group(true, true, true)].into_iter()), + Decision::Replay + ); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ffaff6d80dbf9..6f4a5355d6eb3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -25,6 +25,7 @@ use super::{ kick, qos, queue::{self, admission, compute}, }; +use super::preparation::Admission; use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; @@ -70,20 +71,20 @@ impl Registry { /// registry on the probe stack. No additional allocation is required. pub(super) fn new( clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> impl Init { kernel::try_init!(Self { - qids <- kick::QueueIds::new().chain(|qids| { - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(()) + qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { + qids.reserve_initial_render() }), buffers <- BufferIds::new(), accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new()?, - render: render::RenderState::new(clusters, descriptor_flags)?, + admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, + render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; @@ -121,7 +122,8 @@ impl Registry { /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize) -> Result { + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { if queue.pool_fits(required) { Rebind::Done } else { Rebind::Grow } @@ -129,7 +131,14 @@ impl Registry { Rebind::Fresh }); } + // Physical channels are a device budget, not a permanent entitlement + // of each open logical queue. Only exact idle retirement can yield one. + for queue in self.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { + admission.check()?; + queue.share_idle()?; + } let mut vacant = None; + let teardown = &self.teardown; for queue in self .compute .iter_mut() @@ -137,6 +146,13 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .filter(|queue| queue.reusable()) { + admission.check()?; + // The release of the scheduler these records were bound to is armed + // or not yet consumed: the firmware may process it against this + // queue, which therefore must not change owner until it has. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + continue; + } if queue.pool_vacant() { vacant.get_or_insert(queue.qid()); continue; @@ -167,6 +183,7 @@ impl Registry { } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { + let teardown = &self.teardown; let queue = self .compute .iter_mut() @@ -174,6 +191,10 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid && queue.reusable()) .ok_or(EAGAIN)?; + // Its last owner may have closed while the backing was rebuilt. + if queue.context().is_some_and(|bound| teardown.releasing(bound)) { + return Err(EAGAIN); + } queue.bind_owner(owner, context.clone()) } @@ -267,6 +288,7 @@ impl Registry { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + if !deferred.vacated_room() { break; } if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { deferred.defer_vacated(vacated); freed += 1; @@ -275,6 +297,17 @@ impl Registry { Ok(freed) } + /// Builders and released queues waiting for their exact retirement or pool + /// witness temporarily occupy identities. A failed admission must let those + /// transactions finish rather than fail a healthy successor immediately. + fn compute_admission_error(&self, error: Error) -> Error { + if error == ENOSPC { + EAGAIN + } else { + error + } + } + pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { if owner == 0 || self.compute.iter().flatten().any(|entry| { @@ -292,11 +325,13 @@ impl Registry { .compute .iter() .position(Option::is_none) - .ok_or(ENOSPC)?; + .ok_or_else(|| self.compute_admission_error( + crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; let identity = self.identities.reserve(1)?.start; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; + .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, @@ -373,6 +408,32 @@ pub(super) enum Rebind { Fresh, } + +#[kunit_tests(asahi_g17_uninstalled_cleanup)] +mod uninstalled_cleanup_tests { + use super::*; + use crate::g17::fw::render::Deflake; + use crate::g17::fw::variant::V26A428; + #[test] + fn empty_reservation_prevents_reuse_until_explicit_cancellation() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let first = registry.reserve_compute(1).unwrap(); + assert!(registry.reserved(first).is_ok()); + assert!(registry.reserve_compute(1).is_err()); + let second = registry.reserve_compute(2).unwrap(); + assert_ne!(first.id.qid(), second.id.qid()); + registry.cancel_compute(first).unwrap(); + assert!(registry.reserved(first).is_err()); + let next = registry.reserve_compute(3).unwrap(); + assert_eq!(first.id.qid(), next.id.qid()); + assert!(first.id != next.id); + } +} + /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. pub(super) struct Backend { @@ -380,6 +441,7 @@ pub(super) struct Backend { pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + closed: core::sync::atomic::AtomicBool, } impl Backend { @@ -389,24 +451,54 @@ impl Backend { owner: u64, policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { - let lease = shared.preparations.enter(true)?; - let dev = shared.drm()?; - let (uat, pool, qos, execution, slot) = { + // A closed queue keeps its application context until its scheduler + // release is consumed and the context is returned. When only such + // contexts block this one, wait (bounded) for the teardown instead of + // failing a live client; the preparation lease is not held while + // sleeping, and nothing of the new queue is visible yet. + const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; + let started = now_ns(); + let (lease, uat, pool, qos, execution, slot) = loop { + let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; - let execution = firmware.uat.new_execution_context(vm)?; - ( - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) + match firmware.uat.new_execution_context(vm) { + Ok(execution) => { + break ( + lease, + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + } + Err(error) if error == ENOSPC => { + // Every open queue holds one context; this one is counted + // in `queues()` already but holds none yet. + let open = firmware.queues.admission.queues().saturating_sub(1); + let closing = firmware.uat.execution_contexts_used().saturating_sub(open); + if closing == 0 || now_ns().saturating_sub(started) >= RETAINED_CONTEXT_WAIT_NS { + return Err(super::capacity::refused(super::capacity::Site::ExecutionContexts)); + } + drop(slot); + drop(state); + drop(lease); + // Nothing is visible yet: a signalled creator restarts. + if kernel::current!().signal_pending() { + return Err(ERESTARTSYS); + } + shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } + Err(error) => return Err(error), + } }; + let dev = shared.drm()?; let alloc = super::object::Allocator { dev: &dev, uat: &uat, @@ -429,6 +521,7 @@ impl Backend { context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + closed: core::sync::atomic::AtomicBool::new(false), }, GFP_KERNEL, )?, @@ -438,13 +531,15 @@ impl Backend { /// The shared aperture has exactly one builder. Other queues sleep without /// holding the device mutex; recovery joins its lease before closing roots. - fn shared_compute(&self) -> Result { + fn shared_compute(&self, admission: &Admission<'_>) -> Result { loop { - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); loop { + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -463,7 +558,9 @@ impl Backend { firmware.queues.compute_building = true; break firmware.uat.clone(); } - self.shared.changed.wait(&mut state); + let _ = self.shared.changed.wait_timeout( + &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); } }; let alloc = super::object::Allocator { @@ -474,7 +571,9 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else { built @@ -507,10 +606,12 @@ impl Backend { &self, alloc: &super::object::Allocator<'_>, lease: &super::preparation::Lease, + admission: &Admission<'_>, ) -> Result { { let mut state = self.shared.state.lock(); loop { + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); @@ -522,14 +623,18 @@ impl Backend { firmware.queues.compute_scheduler_building = true; break; } - self.shared.changed.wait(&mut state); + let _ = self.shared.changed.wait_timeout( + &mut state, kernel::time::msecs_to_jiffies(admission.slice_ms()?) + ); } } let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else { built @@ -551,8 +656,23 @@ impl Backend { result } - fn grow_compute(&self, required: usize) -> Result { + /// The caller keeps its preparation lease through this two-phase rollback. + /// An empty registry entry prevents reusing the QID/free-list identity while + /// its old aliases still occupy canonical or client addresses. Recovery joins + /// this lease before removing the session, even if its epoch was invalidated. + fn finish_uninstalled_compute( + &self, retired: Option<(ComputeReservation, KBox)>, + ) -> Result { + let Some((reservation, queue)) = retired else { return Ok(()); }; + drop(queue); + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + firmware.queues.cancel_compute(reservation) + } + + fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { use kernel::time::{delay::fsleep, Delta}; + admission.check()?; { let state = self.shared.state.lock(); let firmware = state.as_ref().ok_or(ENODEV)?; @@ -560,45 +680,79 @@ impl Backend { return Ok(()); } } - let lease = self.shared.preparations.enter(false)?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); let alloc = super::object::Allocator { dev: &dev, uat: &uat }; let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); let mut deferred = DeferredBatch::publication()?; - let started = now_ns(); - loop { + let mut growth = None; + let result = (|| loop { let result = (|| { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; + let queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; if queue.pool_fits(required) { return Ok(None); } - if queue.begin_growth(&firmware.init, required)? { - fence(Ordering::SeqCst); - firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + if growth.is_none() { + let notify = queue.begin_growth(&firmware.init, required)?; + // Capture physical identity before any following fallible + // operation. Close can remove the owner-keyed lookup. + growth = queue.growth_token(); + if notify { + fence(Ordering::SeqCst); + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } } queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) })(); deferred.finish(); match result { Ok(old) => { drop(old); return Ok(()); } - Err(error) if error == EAGAIN && lease.is_current() - && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { + Err(error) if error == EAGAIN && lease.is_current() => { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } Err(error) => return Err(error), } + })(); + if result.is_err() { + if let Some(token) = growth { + let mut retired = None; + let mut state = self.shared.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + for index in 0..firmware.queues.compute.len() { + let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; + let Some(queue) = entry.queue.as_deref_mut() else { continue; }; + if queue.abort_growth(token)? == Some(true) { + // Keep the empty entry and QID reserved until all + // graph aliases have been destroyed outside the lock. + retired = entry.queue.take().map(|queue| (entry.reservation, queue)); + break; + } + } + } + drop(state); + // Tentative replacement mappings also belong to this builder. + // Discard them before a fresh QID can acquire a successor. + drop(alias.take()); + drop(pages.take()); + self.finish_uninstalled_compute(retired)?; + self.shared.queue_events(); + } } + result } - fn build_compute(&self, required: usize) -> Result { + fn build_compute(&self, required: usize, admission: &Admission<'_>) -> Result { + admission.check()?; + self.admission_open()?; loop { - match self.shared_compute() { + match self.shared_compute(admission) { Err(error) if error == EAGAIN => continue, result => { result?; @@ -607,10 +761,12 @@ impl Backend { } } loop { - let lease = self.shared.preparations.enter(false)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let (uat, reservation) = { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if self .shared @@ -619,12 +775,42 @@ impl Backend { { return Err(EIO); } - match firmware.queues.rebind_compute(self.owner, &self.context, required)? { + self.admission_open()?; + let rebound = match firmware.queues.rebind_compute(self.owner, &self.context, required, admission) { + Err(error) if error == ENOSPC => { + drop(state); + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + result => result?, + }; + match rebound { Rebind::Done => return Ok(()), - Rebind::Grow => { drop(state); return self.grow_compute(required); } + Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { - let reservation = firmware.queues.reserve_compute(self.owner)?; + let reservation = match firmware.queues.reserve_compute(self.owner) { + Ok(reservation) => reservation, + Err(error) if error == EAGAIN => { + // The event worker owns release observation and + // drops vacated aliases/backing off this mutex. + drop(state); + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + drop(state); + if reclaimed.is_none() { + return Err(error); + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + }; (firmware.uat.clone(), Ok(reservation)) } } @@ -638,13 +824,32 @@ impl Backend { Err(qid) => { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. - let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - let ring = compute::DescriptorRing::new(&alloc)?; + let built = (|| -> Result<_> { + let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; + Ok((pages, compute::DescriptorRing::new(&alloc)?)) + })(); + let (pages, ring) = match built { + Err(error) if error == ENOSPC => { + drop(lease); + self.wait_compute_capacity(admission)?; + continue; + } + result => result?, + }; let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { continue; } + if let Err(error) = self.admission_open() { drop(state); return Err(error); } + // A competing ready owner may have taken/rearmed it while + // backing was built off-lock. Leave rejected backing local + // so it is dropped after the device guard, not in rearm(). + let ready = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.qid() == qid && queue.can_rearm()); + if !ready { drop(state); continue; } let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, @@ -660,10 +865,12 @@ impl Backend { let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; { let mut state = self.shared.state.lock(); + admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } + self.admission_open()?; firmware.queues.reserved(reservation)?; let (slot, page_list) = pool.pool_descriptor()?; firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; @@ -671,8 +878,15 @@ impl Backend { let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease)?; - let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; + self.compute_scheduler(&alloc, &lease, admission)?; + let queue = compute::Queue::new( + &alloc, + backing, + self.owner, + self.context.clone(), + self.shared.variant.pointers, + self.shared.operand_state, + )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); let (mut built, build_error) = match built { @@ -682,12 +896,14 @@ impl Backend { let result = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { + if let Err(error) = admission.check() { + Err(error) + } else if !lease.is_current() { Err(EAGAIN) } else if let Some(error) = build_error { Err(error) } else { - firmware.queues.install_compute(reservation, &mut built) + admission.check().and_then(|()| firmware.queues.install_compute(reservation, &mut built)) } }; // The old aliases must disappear before a waiting constructor can @@ -703,9 +919,11 @@ impl Backend { .cancel_compute(reservation)?; } drop(lease); - if result != Err(EAGAIN) { - return result; + if result == Err(ENOSPC) { + self.wait_compute_capacity(admission)?; + continue; } + if result != Err(EAGAIN) { return result; } } } } @@ -780,6 +998,9 @@ impl DeferredBatch { drop(release); } } + pub(super) fn vacated_room(&self) -> bool { + self.vacated.len() < self.vacated.capacity() + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -847,7 +1068,7 @@ struct ComputeHost<'a> { } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - self.qids.next_compute_publication() + self.qids.next_publication() } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -918,6 +1139,15 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.owner() == Some(owner)) .ok_or(EIO)?; + // The first command after an owner change waits while the scheduler + // the queue was bound to is being released: a live previous owner may + // have closed after giving up the queue. + if queue + .pending_switch_from() + .is_some_and(|previous| queues.teardown.releasing(previous)) + { + return Err(EAGAIN); + } let mut host = ComputeHost { init: &self.init, primary: &mut self.primary, @@ -1053,12 +1283,14 @@ impl super::Firmware { } impl Backend { - fn ensure_compute(&self, scs_pages: u16) -> Result { + fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { + admission.check()?; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; use kernel::time::{delay::fsleep, Delta}; const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; for _ in 0..SPARED_WAIT_MS { + admission.check()?; let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -1070,7 +1302,10 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - return self.build_compute(required); + match self.build_compute(required, admission) { + Err(error) if error == EAGAIN => continue, + result => return result, + } } // Other owners continue submitting while this owner gives its // queue-local drain witness time to arrive. @@ -1078,6 +1313,7 @@ impl Backend { } // No new command has been accepted. Keep the spared queue and its // retained work intact until a later admission observes its witness. + admission.check()?; Err(EBUSY) } @@ -1089,23 +1325,48 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; let mut recovery_polls = 0; - let mut outer_wait = None; + let mut state_delay = 20i64; + let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); + let check = || { + if packet.cancel_requested() { return Err(ECANCELED); } + self.admission_open()?; + if packet.completion.status().get() != 0 { return Err(EIO); } + Ok(()) + }; + let admission = Admission::new(deadline, &check); loop { - if packet.completion.status().get() != 0 { - return Err(EIO); + admission.check()?; + // All DRM dependencies were satisfied before this scheduler callback. + // Recheck on every retry: pressure may yield an idle binding between + // admission and publication without changing this logical queue. + Backend::ensure_compute(self, scratch.pages(), &admission)?; + admission.check()?; + let lease = self.shared.preparations.enter_until(&admission)?; + // Work states free as this context's earlier commands retire. + // Wait for one under the packet's admission deadline, as render + // publication does, instead of failing the packet. + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => { + drop(lease); + admission.check()?; + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); + continue; + } + result => { + result?; + } } - let lease = self.shared.preparations.enter(false)?; - packet.completion.work_state()?.acquire()?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { drop(state); continue; } - if packet.cancel_requested() { - return Err(ECANCELED); - } + admission.check()?; if self .shared .crashed @@ -1126,7 +1387,13 @@ impl Backend { continue; } firmware.replay_compute(deferred)?; + if firmware.queues.compute(self.owner).is_none() { + drop(state); + drop(lease); + continue; + } let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + admission.check()?; let result = firmware.publish_compute( self.owner, packet.clone(), @@ -1149,11 +1416,6 @@ impl Backend { continue; } if matches!(result, Err(EAGAIN)) && !packet.is_published() { - let now = now_ns(); - let since = *outer_wait.get_or_insert(now); - if now.saturating_sub(since) >= WORK_RING_WAIT_NS { - return Err(EBUSY); - } // Backpressure owns no firmware-visible attempt. Other queues // can submit and the event worker can retire while we sleep. drop(state); @@ -1199,6 +1461,40 @@ impl super::Firmware { } impl super::Shared { + /// Charge only real watchdog expirations, after the per-job allowance. A + /// changing client or recovery generation cannot mint a fresh storm budget. + pub(super) fn watchdog_timeout(self: &Arc) -> bool { + if self.crashed.load(Ordering::Acquire) { return true; } + let stop = { + let mut state = self.state.lock(); + state.as_deref_mut().is_none_or(|firmware| firmware.recovery.timeout_limit(now_ns())) + }; + if stop { self.lose_device_quietly(); } + stop + } + + /// Workload-triggered stop: halt producers first, then join only for a + /// bounded interval. A stalled borrower leaves stopped firmware and all its + /// mappings owned in state. Only a later joined stop may detach/free them. + pub(super) fn stop_runtime(&self) { + let Some(_stopping) = self.stop.try_lock() else { return; }; + { + let mut state = self.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } + firmware.primary.stop(); + } + } + if self.preparations.wait_drained_until(now_ns().saturating_add(WORK_RING_WAIT_NS)).is_err() { + dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); + return; + } + let firmware = self.state.lock().take(); + if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } + } + /// Joins construction and complete host teardown, including RTKit drops. /// The stop-only mutex outlives detached firmware destruction, so unbind /// cannot return while another stop caller still owns mailbox clients. @@ -1211,7 +1507,9 @@ impl super::Shared { let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { - firmware.secondary.stop(); + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } firmware.primary.stop(); } state.take() @@ -1259,12 +1557,23 @@ impl super::Firmware { } } self.queues.observe_pool_releases(&self.init, deferred)?; + if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.pool_release_stalled(now)) { + return Err(ETIMEDOUT); + } if self.render_control_backpressured() { return Ok(None); } + // While the shared lower window refuses free-list backing, no idle + // retained compute pool stays warm: render and new compute need it more. + let cap = if super::freelist::lower_pressure() { + 0 + } else { + COMPUTE_WARM_CAP + }; let (published, next) = self.queues.vacate_idle_pools( &self.init, - COMPUTE_WARM_CAP, + cap, now, COMPUTE_WARM_EXPIRY_NS, )?; @@ -1276,3 +1585,163 @@ impl super::Firmware { } } + +impl Registry { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + self.qids.write_capacity(f)?; + self.admission.write_capacity(f)?; + self.buffers.write_capacity(f)?; + writeln!(f, "qos_ids used={} total=126", self.qos_ids.used())?; + writeln!(f, "metrics_ids used={} total={}", self.metrics.used(), super::fw::buffer::METRICS_SLOTS)?; + self.render.write_capacity(f)?; + writeln!(f, "compute_slots used={} total={} reusable={} building={}", + self.compute.iter().flatten().count(), self.compute.len(), + self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), + self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; + writeln!(f, "compute_channels used={} total={}", + self.compute.iter().flatten().count(), super::kick::COMPUTE_CHANNELS)?; + writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; + for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { + queue.write_capacity(f)?; + } + Ok(()) + } +} + +impl Backend { + pub(super) fn wait_capacity(&self, started: u64) -> Result { + let check = || self.admission_open(); + let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); + self.wait_compute_capacity(&admission) + } + + fn wait_compute_capacity(&self, admission: &Admission<'_>) -> Result { + admission.check()?; + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + // A short reclaim transaction obeys the same session/epoch gate as + // publication. Never keep this lease while sleeping for capacity. + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { + return Ok(()); + } + for queue in firmware.queues.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { + admission.check()?; + queue.share_idle()?; + } + // Consumption, not publication or an elapsed timeout, frees backing. + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + if !firmware.render_control_backpressured() { + let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; + if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } + } + Ok(()) + })(); + deferred.finish(); + self.shared.queue_events(); + match result { + Err(error) if error == EAGAIN => {} + result => result?, + } + admission.check()?; + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + admission.check() + } +} + +#[kunit_tests(asahi_g17_shared_capacity)] +mod shared_capacity_tests { + use super::*; + use crate::g17::fw::render::Deflake; + use crate::g17::fw::variant::V26A428; + + #[test] + fn all_logical_render_owners_fit_beside_shared_compute() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + // Physical compute channels are device-owned and bounded independently + // of how many logical clients have queued unsignalled work. + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); + for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + assert_eq!(leases.len(), crate::mmu::MAX_EXECUTION_CONTEXTS as usize); + // Every accepted logical render owner now has its physical pair; no + // published identity was cancelled or retyped to achieve the fit. + } + + #[test] + fn a_qid_floor_admits_only_the_render_pairs_it_leaves() { + use crate::g17::fw::variant::V25G83; + let limit = super::super::kick::queue_limit(V25G83.qid_floor); + assert_eq!(limit, 42); + let mut registry = KBox::init( + Registry::new(2, Deflake::TWO_CLUSTERS, [0; 2], &V25G83), + GFP_KERNEL, + ) + .unwrap(); + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + for owner in 1..=u64::from(limit) { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + // The next logical queue is refused when it is created, not left waiting for a pair. + assert!(matches!(registry.admission.reserve_queue(), Err(ENOSPC))); + } + + #[test] + fn four_clients_have_capacity_before_any_owner_closes() { + let mut registry = KBox::init( + Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), + GFP_KERNEL, + ) + .unwrap(); + let mut leases = KVec::with_capacity(36, GFP_KERNEL).unwrap(); + for client in 0..4u64 { + for queue in 0..9u64 { + let owner = 1 + client * 9 + queue; + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + } + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + assert_eq!(leases.len(), 36); + } +} + +impl Backend { + /// Logical close is independent of whether an off-lock builder has already + /// installed a physical graph. No constructor can resurrect a closed owner. + fn admission_open(&self) -> Result { + if self.closed.load(Ordering::Acquire) { return Err(ECANCELED); } + if self.context.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if !self.context.is_current() { return Err(EFAULT); } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 3d5f441d8428f..7d690470cd7b5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -18,8 +18,39 @@ impl Backend { } fn stop_after_error(&self) { + // Scheduler/fence callbacks never synchronously join builders or RTKit + // receive work. The event worker performs the bounded runtime stop. self.shared.lose_device_quietly(); - self.shared.stop_queues(); + } + + fn rollback_unpublished_compute(&self) -> Result { + // No outer preparation lease survives compute_packet's return. Hold a + // fresh epoch through alias destruction so recovery can join this work. + loop { + let lease = self.shared.preparations.enter(false)?; + let mut retired = None; + let mut state = self.shared.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { drop(state); drop(lease); continue; } + let mut binding = None; + if let Some(index) = firmware.queues.compute.iter().position(|entry| { + entry.as_ref().and_then(|entry| entry.queue.as_deref()) + .is_some_and(|queue| queue.owner() == Some(self.owner)) + }) { + let entry = firmware.queues.compute[index].as_mut().ok_or(EIO)?; + let (cancel, replaced) = entry.queue.as_deref_mut().ok_or(EIO)? + .rollback_unpublished(self.owner, &self.context)?; + binding = replaced; + if cancel { + retired = entry.queue.take().map(|queue| (entry.reservation, queue)); + } + } + drop(state); + drop(binding); + self.finish_uninstalled_compute(retired)?; + self.shared.queue_events(); + return Ok(()); + } } /// Complete all failure fanout before signalling this packet's unpublished @@ -36,6 +67,11 @@ impl Backend { } deferred.finish(); if !packet.is_published() { + if packet.engine() == Engine::Compute { + if self.rollback_unpublished_compute().is_err() { + self.stop_after_error(); + } + } packet.completion.fail_unpublished(error); } Err(error) @@ -46,6 +82,9 @@ impl Backend { } fn release_queue(&self) -> Result { + // A builder may own only an off-lock reservation when close arrives. + // Record intent before scanning queues; every later install/rearm checks it. + self.closed.store(true, Ordering::Release); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -75,7 +114,11 @@ impl job::Backend for Backend { } fn ensure_compute(&self, scs_pages: u16) -> Result { - Backend::ensure_compute(self, scs_pages) + // A reverse timeline chain must be submitted before the host signal. + // Validate here, but do not consume physical capacity behind unsignalled + // DRM dependencies. compute_packet admits it when the job can run. + crate::g17::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages) + .map(|_| ()) } fn fail_vm(&self, error: Error) { @@ -147,7 +190,17 @@ impl job::Backend for Backend { } fn timed_out(&self, packet: Arc) -> sched::Status { + // A packet still in publication, for example waiting for capacity or + // for its render pair's graph, has nothing visible to firmware and may + // not even have an installed pair. Its publisher owns its bounded wait + // and result, and it keeps its place in the backlog so that a later + // dependency-free command is not published from its ioctl ahead of it. + // It is not a hang and does not count towards the timeout storm limit. + if !packet.is_published() { + return sched::Status::NoHang; + } self.release_backlog(&packet); + if self.shared.watchdog_timeout() { return sched::Status::NoHang; } let result = match packet.engine() { Engine::Compute => self.timeout_compute(&packet), Engine::Render => self.timeout_render(&packet), @@ -177,10 +230,47 @@ impl job::Backend for Backend { if packet.engine() == Engine::Compute { return self.defer_compute_timeout(packet, renewals); } - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .is_some_and(|firmware| firmware.queues.defer_render_timeout(packet, ETIMEDOUT)) + use crate::g17::timeout::{self, Decision}; + // As for compute, a final retirement pass precedes the attribution, and all completion + // callbacks run after releasing the device mutex. Without a collector the attribution + // goes ahead without that pass, as it did before it existed. + let mut deferred = DeferredBatch::worker().ok(); + let result = (|| -> Result { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(false); + }; + if let Some(deferred) = deferred.as_mut() { + firmware.scan_renders(deferred)?; + } + // Borrowed-graph and queued-retirement deferrals share the renewal bound. + if timeout::transaction_grace(renewals) + && firmware.queues.defer_render_timeout(packet, ETIMEDOUT) + { + return Ok(true); + } + let verdict = firmware.queues.render_timeout_decision(packet, renewals); + let decision = verdict.map_or(Decision::Own, |verdict| verdict.decision); + // As compute does: work still blocked behind another VM's older unfinished work + // after the whole grace is innocent, so its failure must stay non-sticky. The pair's + // quarantine extends the class to the pair's other groups. + if decision == Decision::Spare { + packet.completion.classify(crate::g17::recovery::Class::Spared); + } + if let Some(verdict) = verdict { + if let Some(blocker) = verdict.reported_blocker(renewals) { + dev_warn!(self.shared.dev.as_ref(), + "Render watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", + self.owner, verdict.own.vm, verdict.own.published, renewals, decision, + blocker.vm, blocker.published, u8::from(blocker.failed)); + } + } + Ok(decision == Decision::Defer) + })(); + if let Some(deferred) = deferred.as_mut() { + deferred.finish(); + } + matches!(result, Ok(true)) } fn try_early(&self, packet: &Arc) -> bool { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index dac4ca48a34ae..c407d9576e9c4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -27,19 +27,22 @@ impl Backend { let Some(own) = queue.timeout_head() else { return Ok(Decision::Own); }; - let others = firmware.queues.compute.iter().flatten() + let computes = firmware.queues.compute.iter().flatten() .filter_map(|entry| entry.queue.as_deref()) .filter(|other| other.owner() != Some(self.owner)) .filter_map(|other| other.timeout_head()); - let decision = timeout::classify(own, renewals, others); + let others = Iterator::chain(computes, firmware.queues.render_timeout_heads()); + let verdict = timeout::attribute(own, renewals, others); + let decision = verdict.decision; if decision == Decision::Spare { firmware.queues.compute(self.owner).ok_or(EIO)? .classify(&|_| Class::Spared); } - if decision != Decision::Own && (renewals == 0 || decision == Decision::Spare) { + if let Some(blocker) = verdict.reported_blocker(renewals) { dev_warn!(self.shared.dev.as_ref(), - "Compute watchdog owner={} publication={} renewals={} decision={:?} behind earlier work of another VM\n", - self.owner, own.published, renewals, decision); + "Compute watchdog owner={} vm={} publication={} renewals={} decision={:?} behind earlier work of another VM: blocker_vm={} blocker_publication={} blocker_failed={}\n", + self.owner, own.vm, own.published, renewals, decision, + blocker.vm, blocker.published, u8::from(blocker.failed)); } Ok(decision) })(); @@ -50,7 +53,10 @@ impl Backend { /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = (|| { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 0d4a29e779b4a..d71f410c76fb4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -83,7 +83,7 @@ impl Firmware { .ok_or(EIO)? .prepare_replay(&packet)?; let owner = owner.ok_or(EIO)?; - packet.check_dependencies()?; + packet.check_replay_dependencies()?; let dependencies = self.queues.render_prefix(owner, &packet)?; self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) })() }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 3ae1d0a609a33..4fd6e56741986 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -24,14 +24,17 @@ struct Service<'a> { } impl event::Transport for Service<'_> { + fn roles(&self) -> &'static [Role] { + self.firmware.roles() + } fn trace(&mut self, role: Role) { - let firmware = &mut self.firmware; + let firmware = &mut *self.firmware; firmware .rings .drain_trace(&firmware.primary.state.shared.dev, &firmware.init, role); } fn statistics(&mut self, role: Role) { - let firmware = &mut self.firmware; + let firmware = &mut *self.firmware; firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); @@ -263,6 +266,7 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.restart_limit(super::now_ns()) { return Err(ETIMEDOUT); } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, @@ -300,7 +304,7 @@ impl crate::g17::Shared { /// boundary; the collector itself still owns every deferred completion. pub(in crate::g17) fn run_runtime_events(self: &Arc) { if self.crashed.load(core::sync::atomic::Ordering::Acquire) { - self.stop_queues(); + self.stop_runtime(); return; } let mut deferred = { @@ -324,7 +328,7 @@ impl crate::g17::Shared { if let Err(error) = result { dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); self.lose_device_quietly(); - self.stop_queues(); + self.stop_runtime(); } } @@ -345,7 +349,7 @@ impl crate::g17::Shared { }; deferred.finish(); if recover { - self.preparations.wait_drained(); + self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -419,7 +423,7 @@ impl crate::g17::Shared { .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling()); + .any(|queue| queue.retirement_polling() || queue.pool_release_pending()); if polling { self.queue_poll(); } @@ -444,7 +448,8 @@ impl crate::g17::Shared { } else if !recover && !polling && (channel::Events::pending(&firmware.init, Role::Primary)? - || channel::Events::pending(&firmware.init, Role::Secondary)?) + || (firmware.secondary.is_some() + && channel::Events::pending(&firmware.init, Role::Secondary)?)) { // A full bounded pass must not depend on a fresh notification for // records that arrived while it was consuming the same ring. diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a47abf2045526..59db76b9f109a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -24,6 +24,7 @@ use kernel::{prelude::*, sync::Arc}; #[derive(Copy, Clone)] struct Config { clusters: u32, + deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], } @@ -46,7 +47,13 @@ struct Entry { pending_owner_closed: bool, borrowed_active: bool, borrowed_dependencies: Option, + /// `Pair::switch_in_flight` of a borrowed pair. A borrowed pair publishes + /// nothing, so the snapshot can only be stale towards an older switch. + borrowed_switch: Option, event_masks: [u64; 2], + /// A completion event for this pair arrived while it was borrowed. Event + /// passes skip borrowed pairs, so the returning borrower requests another. + missed_event: bool, closed: bool, pending_error: Option<(Arc, Error)>, pending_timeout: bool, @@ -55,6 +62,12 @@ struct Entry { published: bool, } +/// Unpublished objects removed together, then destroyed outside the device lock. +pub(super) struct UnusedRenderSpare { + _entry: Option, + _binding: Option, +} + pub(super) struct RenderState { config: Config, entries: KVec>, @@ -65,7 +78,11 @@ pub(super) struct RenderState { } impl RenderState { - pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + pub(super) fn new( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + ) -> Result { if clusters == 0 { return Err(EINVAL); } @@ -76,6 +93,7 @@ impl RenderState { Ok(Self { config: Config { clusters, + deflake, descriptor_flags, }, entries, @@ -124,6 +142,14 @@ impl RenderState { .ok_or(EIO) } + /// Whether a completion event arrived while the pair in `slot` was + /// borrowed; clears the record. The returned pair is installed again, so + /// the next event pass observes it. + fn take_missed_event(&mut self, slot: u8) -> bool { + self.entry_mut(slot) + .is_ok_and(|entry| core::mem::take(&mut entry.missed_event)) + } + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { let entry = self.entry_mut(reservation.slot)?; if entry.reservation.ids != reservation.ids @@ -157,6 +183,7 @@ impl RenderState { } entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet.clone()); entry.pair.take().ok_or(EIO) } @@ -176,6 +203,7 @@ impl RenderState { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -198,7 +226,114 @@ impl RenderState { } impl Registry { - fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { + /// Yield unused warming only when admission has run out of physical slots. + /// Previously published graphs keep their existing ownership and retirement + /// rules. The caller destroys returned backing after dropping the device lock. + pub(super) fn reclaim_unused_render_spare( + &mut self, + owner: Option, + require_ids: bool, + ) -> Result> { + // Prefer an empty reservation or an already parked channel. + for with_backing in [false, true] { + for slot in 1..RENDER_SLOTS { + let mut warm = false; + let expected = if let Some(entry) = self.render.entries[slot].as_ref() { + if entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.pending_error.is_some() + || entry.pending_timeout + { + continue; + } + // Compute can gain identities only from physical channels + // that never entered command publication. Render can reuse + // a retained render identity without changing its lifetime. + if owner.is_none() + && !entry + .reservation + .ids + .iter() + .all(|id| self.qids.can_cancel_unpublished(*id)) + { + continue; + } + if let Some(pair) = entry.pair.as_ref() { + if !with_backing { + continue; + } + if owner.is_some() && pair.reusable()? { + None + } else if pair.can_yield_unused()? { + warm = true; + Some(entry.reservation.owner) + } else { + continue; + } + } else { + if with_backing || entry.context.is_some() { + continue; + } + None + } + } else { + if with_backing || require_ids || owner.is_none() { + continue; + } + None + }; + if !self + .admission + .reassign_unused_spare(slot as u8, expected, owner) + { + continue; + } + if owner.is_none() { + let entry = self.render.entries[slot].take(); + if let Some(entry) = entry.as_ref() { + for id in entry.reservation.ids { + self.qids.cancel_unpublished(id)?; + } + if let Some(pair) = entry.pair.as_ref() { + self.buffers.unreserve_owner(pair.buffer_id())?; + } + } + return Ok(Some(UnusedRenderSpare { + _entry: entry, + _binding: None, + })); + } + let mut binding = None; + if warm { + let entry = self.render.entries[slot].as_mut().ok_or(EIO)?; + let mut pair = entry.pair.take().ok_or(EIO)?; + self.buffers.unreserve_owner(pair.buffer_id())?; + pair.release_owner(); + let (parked, retired) = KBox::into_inner(pair).park(); + entry.parked = Some(parked); + entry.context = None; + entry.pool = None; + binding = Some(retired); + } + return Ok(Some(UnusedRenderSpare { + _entry: None, + _binding: binding, + })); + } + } + Ok(None) + } + + fn has_render_ids(&self, owner: u64) -> bool { + self.render + .entries + .iter() + .enumerate() + .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) + } + + pub(super) fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); } @@ -228,7 +363,9 @@ impl Registry { pending_owner_closed: false, borrowed_active: false, borrowed_dependencies: None, + borrowed_switch: None, event_masks: [0; 2], + missed_event: false, closed: false, pending_error: None, pending_timeout: false, @@ -262,12 +399,8 @@ impl Registry { { return Err(EBUSY); } - let ids = self.render.entry(slot)?.reservation.ids; - // Physical queue identities survive failed graph construction. Only - // failure to acquire the second identity can return the first one. - for id in ids { - self.qids.publish(id)?; - } + // Construction keeps identities reserved. Publication pins them at the + // first firmware-visible command, so unused warming can still yield. let buffer = self.buffers.allocate_render()?; let entry = self.render.entry_mut(slot)?; entry.reservation.owner = owner; @@ -400,6 +533,11 @@ impl crate::g17::queue::render::Host for Host<'_> { /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + // Firmware builds without MMIO registration install the pair from its queue + // configuration alone. + if !self.firmware.variant.mmio_pair_registration { + return Ok(()); + } if !self.registration_powered { self.firmware.acquire_registration_power()?; self.registration_powered = true; @@ -419,6 +557,9 @@ impl crate::g17::queue::render::Host for Host<'_> { fn publish_qid(&mut self, id: kick::Id) -> Result { self.firmware.queues.qids.publish(id) } + fn next_publication(&mut self) -> Result { + self.firmware.queues.qids.next_publication() + } fn publish_outer_pair( &mut self, priority: u8, @@ -553,6 +694,44 @@ impl crate::g17::Firmware { Ok(()) } + /// A final retirement pass before a render watchdog attributes its timeout, as compute runs + /// scan_compute() first: a head that finished but whose completion the event worker has not + /// processed yet must not defer another VM's timeout. Pairs lent out, or holding a deferred + /// timeout or error, stay with the worker, which observes and settles them in its order. + pub(in crate::g17::runtime) fn scan_renders(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + for slot in 1..self.queues.render.entries.len() { + if !self.queues.render.entries[slot] + .as_ref() + .is_some_and(|entry| { + entry + .pair + .as_ref() + .is_some_and(|pair| pair.published_in_flight()) + && entry.preparing.is_none() + && !entry.growing + && !entry.pending_timeout + && entry.pending_error.is_none() + }) + { + continue; + } + self.retire_render_slot(slot, None, deferred)?; + self.queues.render.collect_retired(slot)?; + self.settle_render_terminal(slot, deferred)?; + } + Ok(()) + } + pub(in crate::g17) fn settle_render_vm( &mut self, status: &Arc, @@ -763,9 +942,19 @@ impl Registry { } pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { for entry in self.render.entries.iter_mut().flatten() { + let mut hit = false; for id in entry.reservation.ids { let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + let bit = masks[qid / 64] & (1u64 << (qid % 64)); + entry.event_masks[qid / 64] |= bit; + hit |= bit != 0; + } + // Event passes skip a pair while preparation, growth or an + // observation has borrowed it. Without a record, a completion + // that arrived meanwhile waits for an unrelated later event or the + // one-millisecond rescan. + if hit && (entry.preparing.is_some() || entry.growing) { + entry.missed_event = true; } } } @@ -816,4 +1005,103 @@ impl Registry { entry.pending_timeout = true; true } + + /// The oldest published group of every render pair, for cross-VM timeout attribution + /// (crate::g17::timeout). + pub(in crate::g17) fn render_timeout_heads( + &self, + ) -> impl Iterator + '_ { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .filter_map(|pair| pair.timeout_head()) + } + + /// A render watchdog blocked behind another VM's older unfinished work gets the same + /// bounded grace as compute; the oldest unfinished work is never excused. When the GPU + /// stalls on one client, the others' renders then outlive the culprit's teardown instead + /// of all timing out together. + pub(in crate::g17) fn render_timeout_decision( + &self, + packet: &Arc, + renewals: u8, + ) -> Option { + let own = self + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.contains_packet(packet)) + .and_then(|pair| pair.timeout_head()); + let computes = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter_map(|queue| queue.timeout_head()); + // Without a head of its own the timeout is its own. + own.map(|own| { + crate::g17::timeout::attribute( + own, + renewals, + Iterator::chain(self.render_timeout_heads(), computes), + ) + }) + } +} + +impl RenderState { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "retired_render_bindings used={} total={}", self.retired.len(), self.retired.capacity())?; + self.pools.write_capacity(f)?; + for (slot, entry) in self.entries.iter().enumerate() { + if let Some(context) = entry.as_ref().and_then(|entry| entry.context.as_ref()) { + context.write_capacity(f, "render", slot)?; + } + } + Ok(()) + } +} + +impl Registry { + /// Give idle private backing to another ready client under capacity pressure. + /// Keep graph/QID/context ownership; the next use rearms the same pool after + /// its consumed release, exactly as eviction-driven reclaim does. + pub(super) fn reclaim_idle_render_pools( + &mut self, init: &crate::g17::initdata::InitData, deferred: &mut DeferredBatch, + ) -> Result { + let render = &mut self.render; + render.pools.observe(init)?; + for entry in render.entries.iter().flatten() { + if !deferred.render_backing_room() { break; } + let Some(pool) = entry.pool.as_ref() else { continue; }; + // None means this pool never acquired a firmware registration; + // otherwise require its exact final release to have been consumed. + if pool.vacant() || !render.pools.released(u32::from(pool.id()), pool.control_va()) + || self.admission.owner_active(entry.reservation.owner) { continue; } + // A sibling or replacement may be building off-lock before it has + // installed its pool reference. Its logical owner also excludes us. + let mut idle = true; + for other in render.entries.iter().flatten().filter(|other| { + other.reservation.owner == entry.reservation.owner + || other.pool.as_ref().is_some_and(|bound| Arc::ptr_eq(bound, pool)) + }) { + if other.constructing || other.growing || other.preparing.is_some() + || other.pending_error.is_some() || other.pending_timeout { + idle = false; + break; + } + if let Some(pair) = other.pair.as_ref() { + if !pair.idle_for_context(pair.context())? { idle = false; break; } + } + } + if idle { + if let Some(backing) = pool.vacate() { deferred.defer_render_backing(backing); } + } + } + Ok(()) + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index b283fb8e7d55c..e3cb174721aaa 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -40,7 +40,7 @@ impl RenderState { entry.published && u32::from(entry.reservation.buffer.0) == request.buffer_slot && entry.context.as_ref().is_some_and(|context| { - request.vm_slot == -1 || request.vm_slot as u32 == u32::from(context.id()) + request.vm_slot == -1 || request.vm_slot as u32 == context.id() }) }) }) @@ -228,6 +228,8 @@ impl crate::g17::Shared { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); entry.growing = true; Some(entry.pair.take().ok_or(EIO)?) } @@ -269,7 +271,10 @@ impl crate::g17::Shared { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.growing = false; + // The grow work item requests an event pass when it finishes. + entry.missed_event = false; } if firmware.queues.render.grow.active != Some(request) { return Err(EIO); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 31bcb688c8001..4ede20d316c64 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -42,6 +42,7 @@ impl crate::g17::Firmware { }; entry.borrowed_active = pair.published_in_flight(); entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.borrowed_switch = pair.switch_in_flight(); entry.preparing = Some(packet); return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); } @@ -60,6 +61,7 @@ impl crate::g17::Firmware { } entry.pair = pair.take(); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; entry.preparing = None; Ok(()) } @@ -152,7 +154,7 @@ impl crate::g17::Shared { drop(state); drop(epoch); self.lose_device_quietly(); - self.stop_queues(); + self.stop_runtime(); if let Some(mut pair) = pair { pair.stopped(&mut |event| { event.finish(); @@ -163,7 +165,7 @@ impl crate::g17::Shared { } let firmware = (*state).as_deref_mut().ok_or(EIO)?; self.changed.notify_all(); - if firmware.render_grow_pending() { + if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } if !epoch.is_current() diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index f7f48e853f225..4713eb5b6b152 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -239,3 +239,11 @@ fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { table.dword(offset + 24)?, ]) } + +impl Pools { + pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "render_pool_leases used={} total={} retained={}", + self.leases.iter().flatten().filter(|lease| !lease.released()).count(), + POOLS - FIRST_RENDER_POOL, self.leases.iter().flatten().count()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 06bea3c84fa7a..e2034c3176595 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -16,6 +16,7 @@ use kernel::{prelude::*, sync::Arc}; struct BuildResources { uat: Arc, objects: Arc, + rings: Arc, ids: Arc, global: crate::mmu::Vm, metrics: Arc, @@ -28,13 +29,14 @@ impl BuildResources { Ok(Self { uat: firmware.uat.clone(), objects: firmware.pool.clone(), + rings: firmware.render_rings.clone(), ids: firmware.render_ids.clone(), global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), metrics_ids: firmware.queues.metrics.clone(), metrics_bases: crate::g17::fw::buffer::MetricsAddresses { firmware: firmware.init.metrics_fw_va()?, - client: crate::hw::t8140::CONFIG.pm_metrics.va, + client: firmware.init.pm_metrics_va(), }, config: firmware.queues.render.config, }) @@ -44,6 +46,10 @@ impl BuildResources { } } +#[expect( + clippy::large_enum_variant, + reason = "Keep preparation allocation-free; boxing adds a fallible allocation to submission." +)] enum Build { Fresh(Reservation, u64, MetricsLease), Parked { @@ -72,8 +78,14 @@ struct BuildContext<'a> { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { - let lease = self.shared.preparations.enter(false)?; + fn build_render_pair( + &self, + slot: u8, + first: &Arc, + optional: bool, + sleep: bool, + ) -> Result { + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); @@ -133,6 +145,13 @@ impl super::super::Backend { if !pair.reusable()? { return Err(EBUSY); } + // The release of the pair's scheduler is armed or still being + // consumed: the firmware may process it against these queues. + // The teardown parks the pair once the release is witnessed; + // the new owner then installs it with fresh queue records. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); if foreign { firmware @@ -164,6 +183,8 @@ impl super::super::Backend { .is_some_and(|pair| pair.published_in_flight()); entry.borrowed_dependencies = entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.borrowed_switch = + entry.pair.as_ref().and_then(|pair| pair.switch_in_flight()); Build::Rebind { pair: entry.pair.take().ok_or(EIO)?, buffer, @@ -238,10 +259,11 @@ impl super::super::Backend { let built = (|| -> Result> { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; Pair::new( - &alloc, + alloc, &resources.objects, + &resources.rings, reservation.ids, slot, self.owner, @@ -252,7 +274,9 @@ impl super::super::Backend { pm_generation, first, resources.config.clusters, + resources.config.deflake, resources.config.descriptor_flags, + self.shared.variant, ) })(); let (mut built, error) = match built { @@ -277,7 +301,7 @@ impl super::super::Backend { built .as_mut() .ok_or(EIO)? - .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; + .finish_install(alloc, firmware.init.pb_descriptor_table()?)?; firmware.queues.install_render(reservation, &mut built) })(); // Fresh backing disappears before its temporary PB reference returns. @@ -316,9 +340,9 @@ impl super::super::Backend { let built = (|| -> Result<_> { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; let prepared = parked.prepare( - &alloc, + alloc, &resources.objects, self.owner, self.context.clone(), @@ -360,7 +384,7 @@ impl super::super::Backend { firmware.queues.buffers.can_release(buffer.0)?; prepared.as_mut().ok_or(EIO)?.finish_install( &parked, - &alloc, + alloc, firmware.init.pb_descriptor_table()?, )?; let pair = parked.commit(&mut prepared, storage)?; @@ -397,6 +421,9 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } firmware.render_grow_ready() }; self.shared.changed.notify_all(); @@ -429,9 +456,9 @@ impl super::super::Backend { let prepared = (|| { let free_list = self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; + .render_pool(alloc, &resources.ids, &resources.global)?; pair.prepare_bind( - &alloc, + alloc, &resources.objects, self.owner, self.context.clone(), @@ -463,10 +490,14 @@ impl super::super::Backend { if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { return Err(ECANCELED); } + // That owner may have closed while this binding was prepared. + if firmware.queues.teardown.releasing(pair.context()) { + return Err(EBUSY); + } if firmware.queues.render.retired.len() == firmware.queues.render.retired.capacity() { - return Err(ENOSPC); + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::RetiredRenderBindings)); } firmware.queues.teardown.displace( pair.context(), @@ -475,7 +506,7 @@ impl super::super::Backend { )?; pair.bind_owner( prepared.as_mut().ok_or(EIO)?, - &alloc, + alloc, firmware.init.pb_descriptor_table()?, )?; if let Some((id, _)) = buffer { @@ -499,6 +530,10 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; + entry.borrowed_switch = None; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } (result, firmware.render_grow_ready()) }; drop(prepared); @@ -556,8 +591,8 @@ impl super::super::Backend { firmware.queues.render.pools.warm(&firmware.init, &pool) } - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet, false)?; + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { + self.build_render_pair(slot, packet, false, sleep)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -568,10 +603,12 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { + // The host lease excludes capacity reclamation until this off-lock + // construction and its final device-locked install have finished. // An optional sibling may be borrowed or waiting for recovery. // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling, packet, true) { - Err(error) if error == EBUSY || error == EAGAIN => {} + match self.build_render_pair(sibling.slot(), packet, true, sleep) { + Err(error) if error == EBUSY || error == EAGAIN || error == super::submit::FALLBACK => {} result => result?, } } @@ -583,8 +620,9 @@ impl super::super::Backend { slot: u8, packet: &Arc, deferred: &mut super::super::DeferredBatch, + sleep: bool, ) -> Result { - let lease = self.shared.preparations.enter(false)?; + let lease = self.enter_preparation(sleep)?; let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); @@ -660,6 +698,9 @@ impl super::super::Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.take_missed_event(slot) { + self.shared.queue_events(); + } if firmware.queues.render.entry(slot)?.closed { firmware .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs index ce9771778acd4..499e3858fcca1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -2,8 +2,8 @@ //! Republication of render groups that firmware retired without executing. //! The pair proves the abandonment (retired, unlinked, stamps covering it, no -//! recovery, fragment end never written) and parks the packet; this pass -//! prepares and publishes the same command again under the device mutex. +//! recovery, fragment end never written) and parks the packets in order; this +//! pass prepares and publishes the same commands again under the device mutex. use super::super::DeferredBatch; use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; @@ -27,79 +27,91 @@ impl Firmware { return Ok(()); } for slot in 0..self.queues.render.entries.len() { - let Some((packet, owner, qids, closed, busy)) = - self.queues.render.entries[slot].as_ref().and_then(|entry| { - let pair = entry.pair.as_ref()?; - let packet = pair.replay_front()?; - Some(( - packet, - entry.reservation.owner, - pair.qids(), - entry.closed, - entry.preparing.is_some() - || entry.growing - || entry.pending_timeout - || entry.pending_error.is_some(), - )) - }) - else { - continue; - }; - if busy { - continue; + // Publish the parked groups front first; a pause keeps the rest parked. + while self.replay_render_front(slot, deferred)? {} + } + Ok(()) + } + + /// Publishes the oldest parked group of `slot`; returns whether it was published. + fn replay_render_front(&mut self, slot: usize, deferred: &mut DeferredBatch) -> Result { + let Some((packet, owner, qids, closed, busy, stalled)) = + self.queues.render.entries[slot].as_ref().and_then(|entry| { + let pair = entry.pair.as_ref()?; + let packet = pair.replay_front()?; + Some(( + packet, + entry.reservation.owner, + pair.qids(), + entry.closed, + entry.preparing.is_some() + || entry.growing + || entry.pending_timeout + || entry.pending_error.is_some(), + pair.replay_stalled(crate::g17::now_ns()), + )) + }) + else { + return Ok(false); + }; + if stalled { + return Err(ETIMEDOUT); + } + if busy { + return Ok(false); + } + if closed || packet.cancel_requested() || !packet.context.is_current() { + self.with_render(slot, |pair, _| { + pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + })?; + return Ok(false); + } + if packet.completion.status().get() != 0 { + self.with_render(slot, |pair, _| { + pair.fail_replay(EIO, &mut |event| deferred.push(event)) + })?; + return Ok(false); + } + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let dev = self.primary.state.shared.drm()?; + let uat = self.uat.clone(); + let prepared = self.with_render(slot, |pair, _| { + if !pair.room() { + return Err(EBUSY); } - if closed || packet.cancel_requested() || !packet.context.is_current() { + let alloc = Allocator { + dev: &dev, + uat: &uat, + }; + pair.prepare(&alloc, &packet, addresses) + }); + match prepared { + Ok(()) => {} + Err(error) if error == EBUSY || error == EAGAIN => return Ok(false), + Err(error) => { self.with_render(slot, |pair, _| { - pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + pair.fail_replay(error, &mut |event| deferred.push(event)) })?; - continue; + return Ok(false); + } + } + match self.publish_render(owner, slot as u8, &packet, deferred) { + Ok(()) => { + self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; + Ok(true) } - if packet.completion.status().get() != 0 { + Err(error) if error == EBUSY || error == EAGAIN => Ok(false), + Err(error) => { self.with_render(slot, |pair, _| { - pair.fail_replay(EIO, &mut |event| deferred.push(event)) + pair.fail_replay(error, &mut |event| deferred.push(event)) })?; - continue; - } - let addresses = RenderAddresses { - stamps: [ - self.init.stamp_addresses(qids[0])?, - self.init.stamp_addresses(qids[1])?, - ], - }; - let dev = self.primary.state.shared.drm()?; - let uat = self.uat.clone(); - let prepared = self.with_render(slot, |pair, _| { - if !pair.room() { - return Err(EBUSY); - } - let alloc = Allocator { - dev: &dev, - uat: &uat, - }; - pair.prepare(&alloc, &packet, addresses) - }); - match prepared { - Ok(()) => {} - Err(error) if error == EBUSY || error == EAGAIN => continue, - Err(error) => { - self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) - })?; - continue; - } - } - match self.publish_render(owner, slot as u8, &packet, deferred) { - Ok(()) => { - self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; - } - Err(error) if error == EBUSY || error == EAGAIN => continue, - Err(error) => { - self.with_render(slot, |pair, _| { - pair.fail_replay(error, &mut |event| deferred.push(event)) - })?; - } + Ok(false) } } - Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 2e88848b93216..24c724cb32470 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -11,7 +11,11 @@ use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; impl Backend { pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; + // Without a collector the watchdog expires again and retries: memory + // pressure must not stop the runtime for every client. + let Ok(mut deferred) = DeferredBatch::worker() else { + return Ok(()); + }; let result = { let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -33,7 +37,7 @@ impl Backend { kernel::bindings::dma_fence_wait_timeout( fence.raw(), false, - kernel::time::msecs_to_jiffies(2000) as _, + kernel::time::msecs_to_jiffies(2000) as isize, ) }; if completed > 0 { @@ -121,6 +125,14 @@ impl crate::g17::Firmware { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_packet()); if let Some(packet) = packet { + // Unclassified work counts as innocent, and a blamed group must never be replayed. + // Work an earlier recovery spared is collateral here (classify() blames before its + // collateral check) and stays spared, so its failure stays non-sticky; the blame + // itself is recorded apart from the class, and refuses any replay of it. + packet.completion.note_blamed(); + if !packet.completion.spared() { + packet.completion.classify(crate::g17::recovery::Class::Guilty); + } self.fail_render_packet(&packet, error, deferred)?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 13381b62da86c..b77fc13b7ed2f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -14,7 +14,29 @@ use kernel::{ time::{delay::fsleep, Delta}, }; +/// Internal result of a publication attempted from the submitting ioctl that +/// met contention before anything became visible: the scheduler publishes the +/// command instead. Never returned to userspace. +pub(super) const FALLBACK: Error = EINPROGRESS; + impl Backend { + /// The submitting ioctl holds the logical queue's mutex and never sleeps + /// for contention or a closed preparation gate: it returns FALLBACK, and + /// the scheduler's worker publishes the command and owns the waits. + pub(super) fn enter_preparation(&self, sleep: bool) -> Result { + if sleep { + return self.shared.preparations.enter(false); + } + // A closed gate (recovery) is the only case enter() would sleep for. + self.shared.preparations.try_enter().map_err(|error| { + if error == EAGAIN { + FALLBACK + } else { + error + } + }) + } + pub(in crate::g17) fn render_packet( &self, packet: &Arc, @@ -23,6 +45,128 @@ impl Backend { self.render_with_lease(packet, None, deferred) } + /// Reserve identities before attaching a lease to the packet. An empty + /// logical slot must not hide reusable physical channels when the shared + /// queue-ID namespace is full. + fn acquire_render_with_ids( + &self, + packet: &Arc, + mut next: Option, + sleep: bool, + ) -> Result { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + let started = super::super::now_ns(); + let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + loop { + self.admission_open()?; + if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = if let Some(lease) = next.take() { + lease + } else { + // Admission sleeps for a pick; the ioctl leaves that to the scheduler. + if !sleep { + return Err(FALLBACK); + } + match pool.acquire_render(self.owner, Request::of(packet), deadline_ns, || { + // Atomics only: the callback runs under the pool mutex. + if packet.cancel_requested() || self.closed.load(Ordering::Acquire) { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + Ok(()) + }) { + Ok(lease) => lease, + Err(error) if error == ENOSPC => { + let reclaimed = { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .reclaim_unused_render_spare(Some(self.owner), false)? + }; + if reclaimed.is_none() { + self.wait_render_capacity(started, sleep)?; + } + drop(reclaimed); + continue; + } + Err(error) => return Err(error), + } + }; + let slot = lease.slot(); + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + match firmware.queues.ensure_render_ids(slot, self.owner) { + Ok(()) => return Ok(lease), + Err(error) if error == ENOSPC => { + let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; + if reclaimed.is_some() { + drop(state); + drop(reclaimed); + next = Some(lease); + continue; + } + // ensure_render_ids leaves no partial entry on refusal. + if firmware.queues.render.entries[usize::from(slot)].is_some() { + return Err(error); + } + if !lease.cancel_empty(self.owner) { + continue; + } + let reclaimed = firmware + .queues + .reclaim_unused_render_spare(Some(self.owner), true)?; + let ready = firmware.queues.has_render_ids(self.owner); + drop(state); + if reclaimed.is_none() && !ready { + self.wait_render_capacity(started, sleep)?; + } + drop(reclaimed); + } + Err(error) => return Err(error), + } + } + } + + /// A pair is borrowed, still settling, or waiting for the release of the + /// scheduler it last ran for to be consumed and the pair parked. The event + /// worker observes both, so kick it on every poll; time out only after the + /// same bound as a capacity wait. + fn wait_render_busy(&self, since: &mut Option, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } + let now = super::super::now_ns(); + if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { + return Err(ETIMEDOUT); + } + self.shared.queue_events(); + fsleep(Delta::from_micros(200)); + Ok(()) + } + + /// A closing owner may still hold its pair until exact retirement. Wait + /// outside both the device mutex and preparation gate so the event worker + /// and recovery can return capacity. No packet is firmware-visible here. + fn wait_render_capacity(&self, started: u64, sleep: bool) -> Result { + if !sleep { + return Err(FALLBACK); + } + self.wait_capacity(started) + } + fn render_with_lease( &self, packet: &Arc, @@ -30,6 +174,7 @@ impl Backend { deferred: &mut DeferredBatch, ) -> Result { packet.check_dependencies()?; + self.admission_open()?; if packet.cancel_requested() { return Err(ECANCELED); } @@ -37,22 +182,17 @@ impl Backend { return Err(EIO); } let interruptible = early.is_none(); - let admission = match early { - Some(lease) => lease, - None => { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; - pool.acquire_render(self.owner, Request::of(packet))? - } - }; + // A publication from the submitting ioctl (an early lease) never + // sleeps for contention; it falls back to the scheduler instead. + let sleep = early.is_none(); + let admission = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; let mut state_polls = 0; let mut state_delay = 20i64; - let mut takeover_polls = 0; + let mut busy_since = None; let mut recovery_polls = 0; + let capacity_started = super::super::now_ns(); loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -65,7 +205,11 @@ impl Backend { return Err(EFAULT); } let work_state = { - let preparation = self.shared.preparations.enter(interruptible)?; + let preparation = if sleep { + self.shared.preparations.enter(interruptible)? + } else { + self.enter_preparation(false)? + }; let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result @@ -76,6 +220,9 @@ impl Backend { match work_state { Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { + if !sleep { + return Err(FALLBACK); + } if state_polls == 1000 { return Err(ETIMEDOUT); } @@ -88,19 +235,23 @@ impl Backend { result?; } } - match self.ensure_render(packet, slot) { + match self.ensure_render(packet, slot, sleep) { + Err(error) if error == ENOSPC => { + self.wait_render_capacity(capacity_started, sleep)?; + continue; + } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since, sleep)?; continue; } result => result?, } - match self.prepare_render(slot, packet, deferred) { + match self.prepare_render(slot, packet, deferred, sleep) { + Err(error) if !packet.is_published() && error == ENOSPC => { + self.wait_render_capacity(capacity_started, sleep)?; + continue; + } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { let recovering = { let state = self.shared.state.lock(); @@ -109,6 +260,9 @@ impl Backend { || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 }; if recovering { + if !sleep { + return Err(FALLBACK); + } if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { return Err(ETIMEDOUT); } @@ -116,11 +270,9 @@ impl Backend { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); + self.wait_render_busy(&mut busy_since, sleep)?; + } else if !sleep { + return Err(FALLBACK); } } result => return result, @@ -163,6 +315,16 @@ impl Backend { } }; let result = self.render_with_lease(packet, Some(admission), &mut deferred); + if matches!(result, Err(error) if error == FALLBACK) && !packet.is_published() { + // Contention before anything became visible: the ioctl does not + // sleep. Return the lease and the claim; the scheduler publishes + // the command, and later ones wait behind it in the backlog. + let lease = packet.completion.detach_render_lease(); + deferred.finish(); + drop(lease); + packet.abandon_publication(); + return false; + } let result = self .finish_publication(packet, result, &mut deferred) .map(|_| ()); @@ -296,6 +458,14 @@ impl crate::g17::Firmware { } let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + // An owner change taken from a live owner which closed meanwhile: its + // first command waits until that scheduler's release has been consumed. + if pair + .owner_switch_from(&packet.context) + .is_some_and(|scheduler| self.queues.teardown.releasing_scheduler(scheduler)) + { + return Err(EBUSY); + } let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index 759c0c2e43dc0..eb04f80361955 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -17,6 +17,15 @@ impl RenderState { } pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + // A pair that changed owner keeps running this scheduler's queues until + // the new owner's first command leaves it, whichever context owns it now. + let scheduler = Some(context.scheduler_va()); + if self.entries.iter().flatten().any(|entry| match entry.pair.as_ref() { + Some(pair) => pair.switch_in_flight() == scheduler, + None => (entry.preparing.is_some() || entry.growing) && entry.borrowed_switch == scheduler, + }) { + return Ok(false); + } for entry in self.entries.iter().flatten() { if !entry .context @@ -120,7 +129,9 @@ impl RenderState { buffers.unreserve_owner(pair.buffer_id())?; let target = out.binding_slot()?; let pair = entry.pair.take().ok_or(EIO)?; - let (parked, retired) = KBox::into_inner(pair).park(); + let (mut parked, retired) = KBox::into_inner(pair).park(); + // This park follows the consumed release of `context`. + parked.release_consumed(context.scheduler_va()); *target = Some(retired); entry.parked = Some(parked); // Detached backing retains these references until the worker runs. diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 8a637e0b3ed01..f8020e8962f84 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -116,7 +116,7 @@ impl Host for Service<'_> { .iter() .flatten() .filter_map(|entry| entry.queue.as_ref()) - .all(|queue| queue.idle_for_context(context)) + .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { self.render.teardown_pools_released(context) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 1a25cd7e3fa3b..53518a76e91a4 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -75,6 +75,21 @@ impl Pending { self.entries.is_empty() } + /// The context's scheduler release is armed or published, and the queues it + /// last ran have not been detached yet. Until then the firmware may still + /// process the release against those queues: none of them may change owner. + pub(crate) fn releasing(&self, context: &Arc) -> bool { + self.find(context) + .is_some_and(|index| self.entries[index].phase != Phase::Deferred) + } + + /// `releasing` for the context that owns scheduler page `scheduler`. + pub(crate) fn releasing_scheduler(&self, scheduler: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && entry.context.scheduler_va() == scheduler + }) + } + fn find(&self, context: &Arc) -> Option { self.entries .iter() diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index e76dab07f56db..d951bd164d6e9 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -6,11 +6,62 @@ pub(crate) const MAX_RENEWALS: u8 = 8; +/// A borrowed render graph or queued off-lock retirement is not evidence of +/// progress. Share the finite deferral bound with cross-VM attribution. +pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS } + +/// Keep the scheduler poll, control-ring waits and ordinary jobs at two seconds. +/// Native ray traversal and AS builds can exceed one poll (CTS stress). Four +/// extra polls give it ten seconds, still one tenth of the legacy scheduler's +/// fallback. This is an execution allowance, not evidence of GPU progress, and +/// is consumed once per scheduler job, independently of cross-VM/replay grace. +pub(crate) const POLL_MS: u32 = 2000; +const EXECUTION_RENEWALS: u8 = 4; + +/// A background control/replay poll has no scheduler fence to expire it. +/// Its timestamp belongs to the exact transaction and is never renewed by work. +pub(crate) fn background_stalled(since: Option, now: u64) -> bool { + since.is_some_and(|since| now.saturating_sub(since) >= 10_000_000_000) +} + +/// Input comes only from fully validated commands. Render metadata covers the +/// VDM half (vertex/tessellation/geometry) and the fragment half independently. +/// AS builders do not traverse rays and need an explicit, bounded UAPI hint. +pub(crate) fn select(compute_ray: bool, render_ray: [bool; 2], as_build: bool) -> bool { + compute_ray || render_ray.into_iter().any(|enabled| enabled) || as_build +} + +/// Firmware dependencies can hold an untagged helper behind a long command on +/// the other engine. Classify the complete validated ioctl before enqueueing +/// anything; no state or allowance leaks into a later submission. +pub(crate) fn select_submission(commands: impl Iterator) -> bool { + commands.into_iter().any(|enabled| enabled) +} + +/// Only the explicit AS-build bit is newly accepted; reserved and unrelated +/// flags remain invalid on G17P. Return None before any command is published. +pub(crate) fn as_build(flags: u32) -> Option { + const AS_BUILD: u32 = 1 << 2; + if flags & !AS_BUILD != 0 { None } else { Some(flags & AS_BUILD != 0) } +} + +pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) -> bool { + if !published || !enabled || *renewals >= EXECUTION_RENEWALS { + return false; + } + *renewals += 1; + true +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { - pub(crate) vm: usize, + /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. + pub(crate) vm: u64, pub(crate) published: u64, pub(crate) finished: bool, + /// Its queue already failed (quarantined) and holds the GPU only until the firmware drains + /// it. No later watchdog of its own will settle it. + pub(crate) failed: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -20,23 +71,156 @@ pub(crate) enum Decision { Spare, } -pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { - if own.finished || !others.into_iter().any(|other| { - other.vm != own.vm && !other.finished && other.published < own.published - }) { - Decision::Own - } else if renewals < MAX_RENEWALS { - Decision::Defer - } else { - Decision::Spare +/// Another VM's older unfinished work that can block `own`: live work before work whose queue +/// already failed, the oldest of each first. +pub(crate) fn blocker(own: Head, others: impl Iterator) -> Option { + others + .filter(|other| other.vm != own.vm && !other.finished && other.published < own.published) + .min_by_key(|other| (other.failed, other.published)) +} + +/// Work behind another VM's older unfinished work is renewed up to MAX_RENEWALS times, so that +/// the blocker's own failure and teardown can free the GPU. After that it is spared only behind +/// a live blocker, which its own watchdog still settles. A blocker that already failed waits only +/// for the firmware to drain it and no longer excuses later work: otherwise an innocent frame +/// published before the culprit, which timed out first, would spare the culprit, and a pair that +/// never drains would spare every later timeout. +pub(crate) fn decide(own: Head, renewals: u8, blocker: Option) -> Decision { + match blocker { + _ if own.finished => Decision::Own, + None => Decision::Own, + Some(_) if renewals < MAX_RENEWALS => Decision::Defer, + Some(blocker) if blocker.failed => Decision::Own, + Some(_) => Decision::Spare, + } +} + +/// A watchdog's decision with the heads it rests on, for its log line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Verdict { + pub(crate) decision: Decision, + pub(crate) own: Head, + pub(crate) blocker: Option, +} + +impl Verdict { + /// The blocker a watchdog line names: at the first deferral and once the grace has run out. + pub(crate) fn reported_blocker(&self, renewals: u8) -> Option { + self.blocker.filter(|_| renewals == 0 || self.decision != Decision::Defer) + } +} + +pub(crate) fn attribute(own: Head, renewals: u8, others: impl Iterator) -> Verdict { + let blocker = blocker(own, others); + Verdict { + decision: decide(own, renewals, blocker), + own, + blocker, } } -#[cfg(test)] +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_execution_allowance))] mod tests { use super::*; - fn head(vm: usize, published: u64, finished: bool) -> Head { - Head { vm, published, finished } + #[test] + fn selects_compute_both_render_halves_and_builds() { + assert!(!select(false, [false; 2], false)); + assert!(select(true, [false; 2], false)); + assert!(select(false, [true, false], false)); + assert!(select(false, [false, true], false)); + assert!(select(false, [true; 2], false)); + assert!(select(false, [false; 2], true)); + } + #[test] + fn helpers_share_only_the_current_submission_allowance() { + assert!(select_submission([false, true, false].into_iter())); + assert!(select_submission([true, false].into_iter())); + assert!(select_submission([false, true].into_iter())); + assert!(!select_submission([false, false].into_iter())); + assert!(!select_submission([].into_iter())); + let mut helper = 0; + for _ in 0..4 { assert!(renew_execution(true, true, &mut helper)); } + assert!(!renew_execution(true, true, &mut helper)); + assert!(!renew_execution(true, false, &mut 0)); + } + #[test] + fn build_hint_rejects_reserved_and_unknown_flags() { + assert_eq!(as_build(0), Some(false)); + assert_eq!(as_build(1 << 2), Some(true)); + for bit in 0..32 { + if bit == 2 { continue; } + assert_eq!(as_build(1 << bit), None); + assert_eq!(as_build((1 << bit) | (1 << 2)), None); + } + assert_eq!(as_build(u32::MAX), None); + } + #[test] + fn borrowed_render_and_queued_retirement_cannot_defer_forever() { + for count in 0..=u8::MAX { + assert_eq!(transaction_grace(count), count < MAX_RENEWALS); + } + } + #[test] + fn background_control_and_replay_polls_have_an_absolute_deadline() { + let since = 42; + assert!(!background_stalled(None, u64::MAX)); + assert!(!background_stalled(Some(since), since - 1)); + assert!(!background_stalled(Some(since), since + 9_999_999_999)); + for _ in 0..1024 { + assert!(background_stalled(Some(since), since + 10_000_000_000)); + } + assert!(background_stalled(Some(since), u64::MAX)); + } + fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { + attribute(own, renewals, others).decision + } + fn head(vm: u64, published: u64, finished: bool) -> Head { + Head { vm, published, finished, failed: false } + } + fn failed(vm: u64, published: u64) -> Head { + Head { vm, published, finished: false, failed: true } + } + #[test] + fn ray_query_execution_is_bounded_at_ten_seconds() { + let mut renewals = 0; + for poll in 1..=4 { + assert!(renew_execution(true, true, &mut renewals)); + assert_eq!(renewals, poll); + } + for _ in 0..256 { + assert!(!renew_execution(true, true, &mut renewals)); + } + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 10_000); + } + #[test] + fn ordinary_and_unpublished_jobs_keep_the_first_deadline() { + for (published, enabled) in [(false, false), (false, true), (true, false)] { + let mut renewals = 0; + assert!(!renew_execution(published, enabled, &mut renewals)); + assert_eq!(renewals, 0); + } + } + #[test] + fn ray_query_and_cross_vm_credits_are_independent_and_finite() { + let mut ray = 0; + let own = head(5, 101, false); + for _ in 0..4 { + assert!(renew_execution(true, true, &mut ray)); + } + for victim in 0..MAX_RENEWALS { + assert!(!renew_execution(true, true, &mut ray)); + assert_eq!(classify(own, victim, [head(7, 100, false)].into_iter()), Decision::Defer); + } + assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); + assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); + } + #[test] + fn exhausted_or_invalid_credit_never_wraps() { + for value in 4..=u8::MAX { + let mut renewals = value; + assert!(!renew_execution(true, true, &mut renewals)); + assert_eq!(renewals, value); + } } #[test] fn earlier_other_vm_gets_bounded_grace() { @@ -52,6 +236,53 @@ mod tests { assert_eq!(classify(head(7, 100, false), 0, [head(5, 101, false)].into_iter()), Decision::Own); } #[test] + fn a_blocker_that_already_failed_defers_then_never_spares() { + // An innocent frame published before the culprit timed out first and failed; its + // quarantined pair keeps its head until the firmware drains it. + let culprit = head(7, 101, false); + for count in 0..MAX_RENEWALS { + assert_eq!(classify(culprit, count, [failed(5, 100)].into_iter()), Decision::Defer); + } + assert_eq!(classify(culprit, MAX_RENEWALS, [failed(5, 100)].into_iter()), Decision::Own); + assert_eq!(classify(culprit, u8::MAX, [failed(5, 100)].into_iter()), Decision::Own); + // A pair that never drains does not spare any later timeout either. + let drained_never = [failed(5, 100), failed(5, 102)]; + for own in [head(7, 105, false), head(9, 200, false)] { + assert_eq!(classify(own, MAX_RENEWALS, drained_never.into_iter()), Decision::Own); + } + } + #[test] + fn a_live_blocker_still_spares_beside_a_failed_one() { + let own = head(9, 103, false); + let others = [failed(5, 100), head(7, 101, false), failed(8, 102)]; + assert_eq!(blocker(own, others.into_iter()), Some(head(7, 101, false))); + assert_eq!(classify(own, MAX_RENEWALS, others.into_iter()), Decision::Spare); + // Failed or not, a blocker must be older, unfinished and of another VM. + let late = [failed(5, 104), failed(9, 100), Head { finished: true, ..failed(5, 100) }]; + assert_eq!(blocker(own, late.into_iter()), None); + assert_eq!(classify(own, 0, late.into_iter()), Decision::Own); + // Among failed blockers the oldest is reported. + let both_failed = [failed(8, 102), failed(5, 100)]; + assert_eq!(blocker(own, both_failed.into_iter()), Some(failed(5, 100))); + } + #[test] + fn the_watchdog_line_names_the_blocker_at_the_first_deferral_and_the_end() { + let own = head(9, 103, false); + let verdict = attribute(own, 0, [failed(5, 100)].into_iter()); + let expected = Verdict { decision: Decision::Defer, own, blocker: Some(failed(5, 100)) }; + assert_eq!(verdict, expected); + assert_eq!(verdict.reported_blocker(0), Some(failed(5, 100))); + assert_eq!(attribute(own, 1, [failed(5, 100)].into_iter()).reported_blocker(1), None); + let last = attribute(own, MAX_RENEWALS, [failed(5, 100)].into_iter()); + assert_eq!(last.decision, Decision::Own); + assert_eq!(last.reported_blocker(MAX_RENEWALS), Some(failed(5, 100))); + let spared = attribute(own, MAX_RENEWALS, [head(5, 100, false)].into_iter()); + assert_eq!(spared.decision, Decision::Spare); + assert_eq!(spared.reported_blocker(MAX_RENEWALS), Some(head(5, 100, false))); + // Nothing blocks: nothing to report. + assert_eq!(attribute(own, 0, [].into_iter()).reported_blocker(0), None); + } + #[test] fn same_vm_finished_and_later_work_do_not_block() { let own = head(5, 101, false); assert_eq!(classify(own, 0, [head(5, 100, false), head(7, 100, true), head(7, 101, false), head(7, 102, false)].into_iter()), Decision::Own); diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index da43c10cd3bef..b2a6cf6e8bfe3 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -90,10 +90,11 @@ impl Proof { vm: &mmu::Vm, range: Range, window: &UscWindow, + variant: &crate::g17::fw::Variant, payload: &Payload, ) -> Result { let requests = RefCell::new(Requests::new()); - let optimistic = payload.validate(window, &Space::Record(&range, &requests)); + let optimistic = payload.validate(window, variant, &Space::Record(&range, &requests)); let requests = requests.into_inner(); if optimistic.is_ok() && !requests.overflow { if self.covers(vm.mapping_validation_epoch(), &requests) { @@ -107,7 +108,7 @@ impl Proof { self.epoch = None; } // Preserve per-range validation's failure and concurrent-mutation behavior. - payload.validate(window, &Space::Checked(&range, vm)) + payload.validate(window, variant, &Space::Checked(&range, vm)) } } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 3bcf8a9692dcc..72238159d0475 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -21,8 +21,10 @@ use kernel::{ }, // }, error::Result, + new_mutex, prelude::*, sync::aref::ARef, + sync::Mutex, uapi, // }; @@ -40,6 +42,7 @@ use crate::{ AsahiDrmDriver, // }, file, + memacct, mmu, }; @@ -60,6 +63,11 @@ pub(crate) struct AsahiObject { kernel: bool, /// Driver aliases outside the user GPUVA tree; they pin the backing. kernel_mappings: AtomicU64, + /// Memory accounting and process attribution of the held backing. + charge: memacct::Charge, + /// Serializes overlapping timestamp writes through every binding of this BO. + #[pin] + pub(crate) timestamp_lock: Mutex<()>, } /// Type alias for the shmem GEM object type for this driver. @@ -74,6 +82,9 @@ pub(crate) type Object = shmem::Object; /// comes with unevictable, as `mapping_set_inaccessible()` sets it; nothing reads or writes a GEM /// shmem file through its mapping. Releasing idle backing or purging clears the mark, so the /// released pages can be swapped and migrated again. +/// +/// The same transitions charge and uncharge the held pages (see [`memacct`]): a held user object +/// counts towards the resident set of the process that created it. pub(crate) fn set_mapping_held(gem: &Object, held: bool) { let raw = gem.as_raw(); // SAFETY: `gem` keeps the GEM object alive. A shmem-backed object's `filp` and its @@ -99,6 +110,7 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { } else { flags.fetch_and(!inaccessible, Ordering::Relaxed); } + gem.charge.set_held(held); } /// Takes the SG table of `gem`, which holds its pages for the device, and marks the @@ -106,7 +118,7 @@ pub(crate) fn set_mapping_held(gem: &Object, held: bool) { pub(crate) fn held_sg_table<'a>( gem: &'a Object, dev: &'a kernel::device::Device, -) -> Result<&'a kernel::scatterlist::SGTable> { +) -> Result> { let sgt = gem.sg_table(dev)?; set_mapping_held(gem, true); Ok(sgt) @@ -115,9 +127,7 @@ pub(crate) fn held_sg_table<'a>( /// VM-private BOs share their assigned VM's reservation object. Validate this /// before acquiring mapping state; generic GPUVM accepts external shareable BOs. pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { - if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 - && vm.is_extobj(gem) - { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 && vm.is_extobj(gem) { return Err(EINVAL); } Ok(()) @@ -211,6 +221,7 @@ pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, kernel: bool, + charge: memacct::Charge, } /// Create a new kernel-owned GEM object. @@ -229,20 +240,23 @@ pub(crate) fn new_kernel_object_wc(dev: &AsahiDrmDevice, size: usize) -> Result< /// Validate before rounding: wrapping a user size can create a much smaller /// backing than requested (or panic with overflow checks). Kernel callers use /// the same boundary so malformed calculated sizes cannot bypass this check. -fn checked_object_size(size: usize) -> Result { - if size == 0 { return Err(EINVAL); } +pub(crate) fn checked_object_size(size: usize) -> Result { + if size == 0 { + return Err(EINVAL); + } size.checked_add(mmu::UAT_PGMSK) .map(|rounded| rounded & !mmu::UAT_PGMSK) .ok_or(EOVERFLOW) } - fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> Result { + let size = checked_object_size(size)?; let gem = shmem::Object::::new( dev, - checked_object_size(size)?, + size, shmem::ObjectConfig:: { reclaimable_cpu_mappings: false, + pages_no_reclaim: false, map_wc, parent_resv_obj: None, }, @@ -250,6 +264,7 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> flags: 0, exportable: false, kernel: true, + charge: memacct::Charge::kernel(&memacct::LEDGER, size as u64), }, )?; @@ -259,12 +274,17 @@ fn new_kernel_object_mapped(dev: &AsahiDrmDevice, size: usize, map_wc: bool) -> } /// Create a new user-owned GEM object with the given flags. +/// +/// `commit` is the admission of `size` (already rounded by [`checked_object_size`]) for the +/// creating client; the object is attributed to the current process. pub(crate) fn new_object( dev: &AsahiDrmDevice, size: usize, flags: u32, parent_object: Option<&shmem::Object>, reclaim_cpu: bool, + pages_no_reclaim: bool, + commit: memacct::Commit, ) -> Result> { if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() { @@ -276,6 +296,7 @@ pub(crate) fn new_object( checked_object_size(size)?, shmem::ObjectConfig:: { reclaimable_cpu_mappings: parent_object.is_some() && reclaim_cpu, + pages_no_reclaim, map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, parent_resv_obj: parent_object, }, @@ -283,6 +304,7 @@ pub(crate) fn new_object( flags, exportable: parent_object.is_none(), kernel: false, + charge: memacct::Charge::user_current(commit), }, )?; @@ -308,16 +330,22 @@ impl DriverObject for AsahiObject { exportable: args.exportable, kernel: args.kernel, kernel_mappings: AtomicU64::new(0), + charge: args.charge, + timestamp_lock <- new_mutex!((), "Asahi timestamp stores"), }) } /// Objects the DRM core allocates for us (dma-bufs imported from other devices and dumb /// buffers) are shared user objects, like a GEM_CREATE object without flags. + /// + /// Their memory is not counted: an import's backing belongs to, and is charged by, its + /// exporter, and the driver offers no dumb buffers (it has no KMS). fn core_object_args(_dev: &AsahiDrmDevice, _size: usize) -> Result { Ok(AsahiObjConfig { flags: 0, exportable: true, kernel: false, + charge: memacct::Charge::none(), }) } @@ -372,6 +400,11 @@ impl Drop for KernelMappingPin { } impl AsahiObject { + /// Whether CPU mappings use the cacheable WRITEBACK policy. + pub(crate) fn is_writeback(&self) -> bool { + self.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK != 0 + } + pub(crate) fn idle_reclaim_candidate(&self) -> bool { !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 } @@ -439,3 +472,22 @@ extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } #[cfg(not(CONFIG_DRM_ASAHI_G15))] pub(crate) fn prepare_new_object(_gem: &Object) -> Result { Err(EINVAL) } +/// Large user BOs have a cheap ENOMEM fallback instead of per-folio reclaim. +/// Keep ordinary resource allocations and kernel-owned backing on their existing +/// policy. 256 MiB also matches Mesa's maximum reusable BO-cache footprint. +pub(crate) fn user_pages_no_reclaim(size: usize, g17: bool) -> bool { + g17 && size >= 256 * 1024 * 1024 +} + +#[kernel::macros::kunit_tests(asahi_gem_allocation_policy)] +mod allocation_policy_tests { + use super::*; + + #[test] + fn large_user_backing() { + assert!(!user_pages_no_reclaim(256 * 1024 * 1024 - 1, true)); + assert!(user_pages_no_reclaim(256 * 1024 * 1024, true)); + assert!(user_pages_no_reclaim(3_257_113_344, true)); + assert!(!user_pages_no_reclaim(3_257_113_344, false)); + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index a06b5da5ed6b1..be65af433a633 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,7 +11,6 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. -use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -268,6 +267,11 @@ pub(crate) struct GpuManager { /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { + /// Admission bound checked before copying the userspace command stream. + fn max_cmdbuf_size(&self) -> usize { + usize::MAX + } + /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; @@ -285,6 +289,10 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). + fn supports_soft_faults(&self) -> bool { + *crate::module_parameters::fault_control.value() == 0xb + } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false @@ -303,7 +311,7 @@ pub(crate) trait Gpu: Send + Sync { if self.queue_limits().is_some() { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; } - if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { + if self.soft_faults() && self.supports_soft_faults() { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } flags @@ -312,32 +320,39 @@ pub(crate) trait Gpu: Send + Sync { fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; - /// Backend-specific user address aperture; other GPUs keep their geometry's range. + /// The VA range that user VMs may use (`vm_start..vm_end` of GET_PARAMS). fn user_range(&self) -> Range { self.uat_geometry().user_usable_range() } - /// Use tracked command publication and sticky VM status on this backend. + /// Whether this GPU uses the tracked submission model of `file.rs` (besides G17, which + /// `file.rs` recognizes by type): VM_BIND batches are validated as a whole against + /// driver-reserved ranges before any change, and SUBMIT copies the command buffer before + /// resolving syncs because the queue installs a fence in every output sync even on failure. fn tracked_submission(&self) -> bool { false } - /// Whether faults may be reported as recoverable to userspace. + /// Whether SOFT_FAULTS may be advertised when `fault_control` selects soft faults. fn soft_faults(&self) -> bool { true } - /// Populate and clean fresh object backing before any WC CPU mapping. + /// Whether new user objects must be populated, cleaned to the point of coherency and pinned + /// at creation, before any CPU view exists (GPUs whose device is not DMA-coherent and whose + /// default CPU mappings are write-combined). fn prepare_new_objects(&self) -> bool { false } - /// Optional backend-specific upper bound for a VM_BIND batch. + /// The largest VM_BIND batch, if the GPU's UAPI contract bounds it; with a bound, a batch also + /// needs at least one operation, a non-null pointer and a stride that is a multiple of 8 and at + /// most a page (EINVAL otherwise). `None` keeps the shared rules. fn bind_batch_limit(&self) -> Option { None } - /// A userspace VM handle was closed; release backend-only registry references. - fn vm_closed(&self) {} + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; + /// A userspace VM handle was destroyed (VM_DESTROY); release backend-only registry references. + fn vm_destroyed(&self) {} /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 /// T8122 runtime admits any client. @@ -350,6 +365,13 @@ pub(crate) trait Gpu: Send + Sync { } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; + /// Create a VM charged to a DRM file when the backend bounds retained bindings. + fn new_vm_for_file(&self, kernel_range: Range, _file_id: u64) -> Result { + self.new_vm(kernel_range) + } + /// Release the backend's handle owner after closing user mapping admission. + /// Accepted jobs and firmware resource leases retain their own references. + fn vm_closed(&self, _vm: &mmu::Vm) {} /// Create a new user command queue. fn new_queue( &self, @@ -359,12 +381,12 @@ pub(crate) trait Gpu: Send + Sync { priority: u32, usc_exec_base: u64, ) -> Result>; - /// Map a BO as a timestamp buffer + /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( &self, bo: gem::ObjectRef, range: Range, - ) -> Result; + ) -> Result; } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -752,6 +774,7 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, + mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) @@ -1427,15 +1450,15 @@ impl Gpu for GpuManager::ver { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - bo.map_range_into_range( + ) -> Result { + crate::timestamp::Buffer::firmware(bo.map_range_into_range( self.uat.kernel_vm(), range, IOVA_KERN_TIMESTAMP_RANGE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - ) + )?) } } diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index deffe3fe47d0b..2650131a4a293 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -13,6 +13,7 @@ const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; pub(crate) mod t602x; +pub(crate) mod t6040; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; @@ -23,6 +24,7 @@ pub(crate) mod t8140; pub(crate) enum GpuGen { G13 = 13, G14 = 14, + G16 = 16, G17 = 17, } diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs new file mode 100644 index 0000000000000..7a4d6f288dc3d --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t6040 platforms (M4 Pro). +//! +//! The M4 Pro runs the macOS 26.6 GPU firmware (25G83), which has the A18 Pro firmware's +//! interface; [`fw::variant::V25G83`] describes where it differs. + +use crate::g17::fw::{ + self, + initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, +}; +use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; +use crate::hw::{GpuGen, GpuVariant}; +use crate::mmu::UatQuirks; + +/// Static configuration of the T6040 GPU. +pub(crate) const CONFIG: Config = Config { + chip_id: CHIP_ID, + identity: Identity { + family: 0xa, + // ID register 0x0a03_1100: family 0xa, variant 3, revision 0x11 (B1). + variant: 3, + num_dies: 1, + gpu_variant: GpuVariant::S, + usc_generation: 3, + hal_generation: 200, + }, + uat_ias: 42, + uat_oas: 42, + // One coprocessor carries both firmware roles; `pagetables` holds only its root. + secondary_root_offset: None, + crash_buffers: CrashBuffers::FirmwareData, + // Read by the firmware through the kernel lower root, whose top-level entries it only + // samples when it starts. + pm_metrics: FixedBuffer { + va: 0x10_0008_0000, + size: 0x8000, + }, + windows: ®ISTER_WINDOWS, + reserved_slots: &[], + perf: &PERF, + opps: OPPS, + // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The + // rest is the policy the M4 Pro has run with; none of it was tested otherwise. + uat_quirks: UatQuirks { + reserved_tables: true, + clean_tables: true, + page_tlbi: true, + handoff_lock: false, + handoff_slot: 0xffff_ffff, + native_gpu_permissions: true, + }, + region_c: ®ION_C, + adopt_root_entries: false, + fabric_bridge: false, + // The firmware's per-slot recovery keys are not known for this build; recovery relies on + // the firmware's own blame. + recovery_slot_keys: None, + // Userspace drives this GPU as a G16 part. + gpu_generation: GpuGen::G16, + // The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 GPU firmware. + firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { + os_version: [26, 6, 2], + name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", + variant: &fw::variant::V25G83, + }]), + // Measured on the 20-core M4 Pro; other core counts are refused at probe. + core_layout: CoreLayout { + cores: Some((2, 20)), + operand_state: fw::compute::OperandState::TWO_BY_TEN, + deflake: fw::render::Deflake::TWO_CLUSTERS, + }, +}; + +/// Chip identifier written to the hardware data. +pub(crate) const CHIP_ID: u32 = 0x6040; + +/// Base of the firmware MMIO aperture in the kernel window. +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; + +const fn window( + slot: usize, + phys: u64, + offset: u64, + size: u32, + unk_18: u64, + unk_20: u32, +) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + unk_18, + unk_20, + instances: 1, + phys_stride: 0, + } +} + +/// `window` repeated `instances` times, `phys_stride` apart in physical memory. +const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> RegisterWindow { + RegisterWindow { + instances, + phys_stride, + ..window + } +} + +/// Register windows the firmware maps, by register-map slot. Slot 3 holds the four memory +/// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ + window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), + instances(window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), 4, 0x200_0000), + window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), + window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), + window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), + window(11, 0x5_003d_8000, 0x28_4000, 0x4000, 0, 2), + window(12, 0x4_8565_c000, 0x28_c000, 0x4000, 0, 2), + window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), + window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), + window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), + instances(window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), 4, 0x20_0000), + window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), + window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), + window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), + window(29, 0x2_88d1_0000, 0x40_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x2_88d4_0000, 0x40_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x2_88d6_0000, 0x41_0000, 0x2_0000, 0xd6_0000, 2), + window(33, 0x2_88d4_4000, 0x43_4000, 0x4000, 0xd4_4000, 2), + window(34, 0x2_88d4_c000, 0x43_c000, 0x200, 0xd4_c000, 2), + window(35, 0x2_88e0_0000, 0x44_4000, 0x4000, 0xe0_0000, 2), + window(37, 0x2_88e4_0000, 0x44_c000, 0x4000, 0xe4_0000, 2), + window(39, 0x2_88e0_8000, 0x45_4000, 0x8000, 0, 2), + window(40, 0x2_88e1_c000, 0x46_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x2_88e1_f800, 0x46_b800, 0x4000, 0, 2), + window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), +]; + +/// Performance-table configuration. The device tree lists the sixteen operating points in +/// performance-state order, with one voltage per cluster. +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], + states_b: [0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15], + relative_a: [0, 9, 16, 23, 29, 38, 49, 60, 69, 79, 100], + relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + sram_min_mv: 810, +}; + +/// Operating points the performance tables reference. +pub(crate) const OPPS: usize = 16; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < OPPS); + assert!((PERF.states_b[index] as usize) < OPPS); + index += 1; + } +}; + +/// Region C. +const REGION_C: RegionCConfig = RegionCConfig { + unk_54: 0x0fa0, + unk_78: [1, 0, 0], + max_power_mw: v25g83::MAX_POWER_MW, + unk_9c: 0, + unk_a0: 0, + unk_7c4: 1500, + unk_9e0: [ + 1, + 0x0102_0200, + 0x010a_2801, + 0x2802_0101, + 0x0101_010a, + 0x010a_2802, + 0x2801_0101, + 0x0001_010a, + ], +}; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 143ba720d06f2..75925713bf10b 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -4,10 +4,11 @@ use crate::g17::{ Config, + CrashBuffers, FixedBuffer, Identity, // }; -use crate::hw::GpuVariant; +use crate::hw::{GpuGen, GpuVariant}; /// Static configuration of the T8140 GPU. pub(crate) const CONFIG: Config = Config { @@ -23,10 +24,10 @@ pub(crate) const CONFIG: Config = Config { uat_ias: 42, uat_oas: 42, // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. - secondary_root_offset: 0x40000, + secondary_root_offset: Some(0x40000), // The coprocessors request their crash log buffers at these addresses, in this order, and // expect them to be mapped before they start. - crash_buffers: [ + crash_buffers: CrashBuffers::Fixed([ FixedBuffer { va: 0x01e1_c000, size: 0x8000, @@ -35,13 +36,30 @@ pub(crate) const CONFIG: Config = Config { va: 0x01f5_0000, size: 0x8000, }, - ], + ]), // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. pm_metrics: FixedBuffer { va: 0x10_0008_0000, size: 0x8000, }, + recovery_slot_keys: Some(recovery::SLOT_KEYS), + gpu_generation: GpuGen::G17, + windows: ®ISTER_WINDOWS, + reserved_slots: &RESERVED_REGISTER_SLOTS, + perf: &PERF, + opps: OPPS, + uat_quirks: crate::mmu::UatQuirks::DEFAULT, + region_c: &crate::g17::fw::initdata::REGION_C_26A428, + adopt_root_entries: true, + fabric_bridge: true, + firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), + // The layouts the A18 Pro has always run with. + core_layout: crate::g17::CoreLayout { + cores: None, + operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, + deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, + }, }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; @@ -67,6 +85,8 @@ const fn window( size, unk_18, unk_20, + instances: 1, + phys_stride: 0, } } @@ -273,7 +293,7 @@ pub(crate) mod resources { pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MIN: usize = TVB_PAGE as usize; pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; pub(crate) const TPC_MIN: usize = 0x3_0000; pub(crate) const TPC_MAX: usize = 0x200_0000; @@ -384,6 +404,9 @@ pub(crate) mod qos { pub(crate) mod scheduling { use crate::g17::fw::queue::Policy; + /// `DRM_ASAHI_PRIORITY_HIGH`: class 1, as the G13 tables map HIGH; the + /// policy word of the user classes is kept (no macOS trace of class 1 yet). + pub(crate) const HIGH: Policy = Policy::new(1, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_MEDIUM`. pub(crate) const MEDIUM: Policy = Policy::new(2, 2, super::qos::CLASS, super::qos::SHARE); /// `DRM_ASAHI_PRIORITY_LOW`. @@ -393,7 +416,9 @@ pub(crate) mod scheduling { /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; - pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; + // One physical pair per logical owner leaves room for every execution + // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH. + pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; pub(crate) const RENDER_DEPTH: u8 = 16; pub(crate) const COMPUTE_DEPTH: usize = 16; } diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index dc8f1a074e809..bcb9fd32c7d25 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -254,7 +254,8 @@ impl gpu::Gpu for Backend { Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, self.soc.features.fragment_dependency)?,GFP_KERNEL)?) } - fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { - Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range) + fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { + crate::timestamp::Buffer::firmware( + Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range)?) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index ca16e18f10fbf..a60e38bf9addd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -18,12 +18,13 @@ impl Destination { let guard = objects.lock(); let object = guard.get(timestamp.handle.try_into()?).ok_or(ENOENT)?; match &*object { - file::Object::TimestampBuffer(mapping) => { + file::Object::TimestampBuffer(buffer) => { + let mapping = buffer.firmware_mapping()?; let offset = timestamp.offset as usize; if offset & 7 != 0 || offset.checked_add(8).ok_or(EOVERFLOW)? > mapping.size() { return Err(EINVAL); } - Ok(Some(Self { mapping: mapping.clone(), offset })) + Ok(Some(Self { mapping, offset })) } } } diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 7a5587ef6ec78..5f0ab2b4aea4f 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -15,9 +15,39 @@ use kernel::page::PAGE_SIZE; use crate::debug::*; use crate::mmu; +use core::sync::atomic::{AtomicBool, Ordering}; type Asid = u8; +/// Invalidate ranges page by page (`mmu::UatQuirks::page_tlbi`): never range operations. +/// +/// The setting is global because the invalidation helpers take no UAT. It assumes one GPU per +/// system: `Uat::new` sets it for that GPU before the GPU's first invalidation, and a second +/// UAT with another policy would change it for both. +static PAGE_TLBI: AtomicBool = AtomicBool::new(false); + +/// Ranges above this many pages are invalidated by ASID in page mode. +const PAGE_TLBI_MAX_PAGES: usize = 2048; + +/// Selects page-by-page range invalidation for the system's GPU ([`PAGE_TLBI`]). +pub(crate) fn set_page_tlbi(enabled: bool) { + PAGE_TLBI.store(enabled, Ordering::Relaxed); +} + +/// Invalidates `len` bytes at `va` one 16 KiB page at a time, or the whole ASID above +/// [`PAGE_TLBI_MAX_PAGES`]. +fn tlbi_pages(asid: Asid, va: usize, len: usize) { + let start = va >> mmu::UAT_PGBIT; + let end = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; + if end - start > PAGE_TLBI_MAX_PAGES { + tlbi_asid(asid); + return; + } + for page in start..end { + tlbi_page(asid, page << mmu::UAT_PGBIT); + } +} + /// Invalidate the entire GPU TLB. #[inline(always)] pub(crate) fn tlbi_all() { @@ -78,12 +108,16 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } let start_pg = va >> mmu::UAT_PGBIT; let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; const BADDR_MASK: u64 = (1 << 37) - 1; - let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); + let mut val: u64 = (u64::from(asid) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); let pages = end_pg - start_pg; // Guess? It's possible that the page count is in terms of 4K pages @@ -167,6 +201,10 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } if va == 0 || len >= TLBI_RANGE_MAX { tlbi_asid(asid); @@ -185,7 +223,7 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { }; let num = ((len - 1) >> unit_shift) as u64; - let val = ((asid as u64) << 48) + let val = (u64::from(asid) << 48) | (TLBI_TG_16K << 46) | (scale << 44) | (num << 39) diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs new file mode 100644 index 0000000000000..9da09968171e4 --- /dev/null +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -0,0 +1,549 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU memory attribution, accounting and admission limits. +//! +//! The backing of a GEM object that the GPU holds is pinned shmem: it sits on the +//! unevictable LRU and cannot be swapped out. It is charged to the memory cgroup of +//! the task that faulted it in, but the OOM killer scores a task by the resident set +//! of its mm, and GPU-only pages are mapped into no process. A game holding +//! gigabytes of buffers therefore scores like a tiny process, and so does a +//! compositor; which one an OOM kill hits is left to the other terms of the score. +//! +//! This module closes that gap: +//! +//! - Every user object remembers the mm of the process that created it (an `mmgrab` +//! reference, which keeps only the `mm_struct` itself). While the GPU holds the +//! object's backing, its pages are added to that mm's `MM_SHMEMPAGES` counter. +//! They are shmem pages, so this is where `oom_badness()`, the OOM report's +//! `rss_shmem` column and `RssShmem`/`VmRSS` in `/proc//status` look. Pages +//! are uncharged when idle reclaim or purge releases the backing and when the +//! object is freed; the mm reference is dropped after the last uncharge, so the +//! counter is back to its own value before `__mmdrop()` checks it. +//! - Each client (DRM file) and the device keep totals of created and held bytes, +//! reported in fdinfo (`drm-total-system`, `drm-resident-system`) and in the +//! `gpu_memory` sysfs file next to `agx_stats`. +//! - A new user object is admitted only while its client stays within a per-client +//! limit and the device within a device-wide limit, both fractions of RAM. +//! A refused allocation fails with `ENOMEM`, which userspace reports as +//! out-of-device-memory, instead of pushing the machine into a global OOM. +//! +//! Imported dma-bufs are charged to their exporter and are not counted here. +//! Kernel objects (firmware structures, per-queue buffers and the global heaps) +//! count towards the device total and its limit, but are attributed to no process. + +use core::sync::atomic::{ + AtomicBool, + AtomicU64, + Ordering, // +}; + +use kernel::{ + mm::{ + Mm, + RssCounter, // + }, + page::PAGE_SIZE, + prelude::*, + sync::{aref::ARef, Arc}, +}; + +use crate::module_parameters; + +/// Device-wide GPU memory totals, in bytes. +/// +/// The layout is shared with the C sysfs shim (`struct asahi_gpu_memory`). +#[repr(C)] +pub(crate) struct Ledger { + /// Size of every live user object (created, not imported). + pub(crate) user_committed: AtomicU64, + /// Size of the live user objects whose backing the GPU currently holds. + pub(crate) user_resident: AtomicU64, + /// Size of the live kernel objects whose backing the GPU holds. + pub(crate) kernel_resident: AtomicU64, + /// User object creations refused by a limit. + pub(crate) refused: AtomicU64, + /// Device-wide limit on user committed plus kernel resident bytes (0: none). + pub(crate) limit: AtomicU64, + /// Per-client limit on committed bytes (0: none). + pub(crate) client_limit: AtomicU64, +} + +// Keep the Rust half of the C sysfs layout contract checked as well. +const _: () = { + assert!(core::mem::offset_of!(Ledger, refused) == 24); + assert!(core::mem::offset_of!(Ledger, client_limit) == 40); + assert!(core::mem::size_of::() == 48); +}; + +/// The ledger of the (single) AGX GPU. +pub(crate) static LEDGER: Ledger = Ledger::new(); + +/// The limits that follow from the amount of RAM and two percentages. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct Limits { + /// Device-wide limit in bytes (0: none). + pub(crate) device: u64, + /// Per-client limit in bytes (0: none). + pub(crate) client: u64, +} + +fn pct_of(total: u64, pct: u32) -> u64 { + match pct { + 0 => 0, + // Above 100% cannot be held anyway; treat it as "no limit". + p if p >= 100 => 0, + p => total / 100 * u64::from(p) + total % 100 * u64::from(p) / 100, + } +} + +/// Computes the limits for `total_ram` bytes of RAM. +/// +/// A percentage of 0 or 100 and above disables that limit. A per-client limit +/// is never larger than the device limit. +pub(crate) fn limits_for(total_ram: u64, device_pct: u32, client_pct: u32) -> Limits { + let device = pct_of(total_ram, device_pct); + let mut client = pct_of(total_ram, client_pct); + if device != 0 && (client == 0 || client > device) { + client = device; + } + Limits { device, client } +} + +/// Adds `bytes` to `counter` unless the result would exceed `limit` (0: no limit). +/// +/// Returns whether the bytes were added. Concurrent callers cannot overshoot. +fn try_add_within(counter: &AtomicU64, bytes: u64, limit: u64, extra: u64) -> bool { + let mut cur = counter.load(Ordering::Relaxed); + loop { + let Some(new) = cur.checked_add(bytes) else { + return false; + }; + if limit != 0 && new.saturating_add(extra) > limit { + return false; + } + match counter.compare_exchange_weak(cur, new, Ordering::AcqRel, Ordering::Relaxed) { + Ok(_) => return true, + Err(actual) => cur = actual, + } + } +} + +fn sub(counter: &AtomicU64, bytes: u64) { + let prev = counter.fetch_sub(bytes, Ordering::AcqRel); + if prev < bytes { + pr_warn!("asahi: GPU memory accounting underflow ({} < {})\n", prev, bytes); + counter.store(0, Ordering::Relaxed); + } +} + +impl Ledger { + pub(crate) const fn new() -> Self { + Self { + user_committed: AtomicU64::new(0), + user_resident: AtomicU64::new(0), + kernel_resident: AtomicU64::new(0), + refused: AtomicU64::new(0), + limit: AtomicU64::new(0), + client_limit: AtomicU64::new(0), + } + } + + /// Sets the limits from the module parameters and the amount of RAM. + pub(crate) fn configure(&self) { + // SAFETY: `struct sysinfo` is plain integers, for which all zeroes is a valid value. + let mut info: kernel::bindings::sysinfo = unsafe { core::mem::zeroed() }; + // SAFETY: `si_meminfo()` only fills in the structure it is given. + unsafe { kernel::bindings::si_meminfo(&mut info) }; + let total = (info.totalram as u64).saturating_mul(u64::from(info.mem_unit)); + let limits = limits_for( + total, + *module_parameters::mem_limit_pct.value(), + *module_parameters::client_mem_limit_pct.value(), + ); + self.set_limits(limits); + pr_info!( + "asahi: GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", + limits.device >> 20, + limits.client >> 20 + ); + } + + /// Admission capacities exported in the optional GET_PARAMS tail. + pub(crate) fn limits(&self) -> Limits { + Limits { + device: self.limit.load(Ordering::Relaxed), + client: self.client_limit.load(Ordering::Relaxed), + } + } + + pub(crate) fn set_limits(&self, limits: Limits) { + self.limit.store(limits.device, Ordering::Relaxed); + self.client_limit.store(limits.client, Ordering::Relaxed); + } + + /// Admits a new user object of `bytes` for `client`. + /// + /// Reserves the bytes against both limits; the returned [`Commit`] gives + /// them back when it is dropped. + pub(crate) fn admit(&'static self, client: &Arc, bytes: u64) -> Result { + let client_limit = self.client_limit.load(Ordering::Relaxed); + if !try_add_within(&client.committed, bytes, client_limit, 0) { + self.refused.fetch_add(1, Ordering::Relaxed); + return Err(ENOMEM); + } + // Kernel objects share the device limit but are never refused. + let kernel = self.kernel_resident.load(Ordering::Relaxed); + if !try_add_within( + &self.user_committed, + bytes, + self.limit.load(Ordering::Relaxed), + kernel, + ) { + sub(&client.committed, bytes); + self.refused.fetch_add(1, Ordering::Relaxed); + return Err(ENOMEM); + } + Ok(Commit { + ledger: self, + client: client.clone(), + bytes, + }) + } +} + +/// Per-client (DRM file) totals, in bytes. +pub(crate) struct Client { + /// Size of the live user objects this client created. + committed: AtomicU64, + /// Size of those whose backing the GPU currently holds. + resident: AtomicU64, +} + +impl Client { + pub(crate) fn new() -> Result> { + Ok(Arc::new( + Self { + committed: AtomicU64::new(0), + resident: AtomicU64::new(0), + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn committed(&self) -> u64 { + self.committed.load(Ordering::Relaxed) + } + + pub(crate) fn resident(&self) -> u64 { + self.resident.load(Ordering::Relaxed) + } +} + +/// Bytes of a user object reserved against the limits. +pub(crate) struct Commit { + ledger: &'static Ledger, + client: Arc, + bytes: u64, +} + +impl Drop for Commit { + fn drop(&mut self) { + sub(&self.client.committed, self.bytes); + sub(&self.ledger.user_committed, self.bytes); + } +} + +enum Kind { + /// An object the DRM core created (a dma-buf import): charged to its exporter. + None, + /// A driver-owned object. + Kernel(&'static Ledger), + /// A user object, attributed to the process that created it. + User { + commit: Commit, + mm: Option>, + }, +} + +/// The accounting state of one GEM object. +pub(crate) struct Charge { + kind: Kind, + bytes: u64, + /// Whether the bytes are currently counted as held by the GPU. + held: AtomicBool, +} + +// SAFETY: `ARef` may be dropped on any thread and the counters are atomic. +unsafe impl Send for Charge {} +// SAFETY: Shared access only touches atomics and the per-cpu RSS counters. +unsafe impl Sync for Charge {} + +impl Charge { + /// An object that is not counted (a dma-buf import). + pub(crate) fn none() -> Self { + Self { + kind: Kind::None, + bytes: 0, + held: AtomicBool::new(false), + } + } + + /// A kernel object of `bytes`. + pub(crate) fn kernel(ledger: &'static Ledger, bytes: u64) -> Self { + Self { + kind: Kind::Kernel(ledger), + bytes, + held: AtomicBool::new(false), + } + } + + /// A user object admitted by `commit`, attributed to `mm` if there is one. + pub(crate) fn user(commit: Commit, mm: Option>) -> Self { + let bytes = commit.bytes; + let mm = if *module_parameters::oom_attribution.value() != 0 { + mm + } else { + None + }; + Self { + kind: Kind::User { commit, mm }, + bytes, + held: AtomicBool::new(false), + } + } + + /// A user object attributed to the current process. + pub(crate) fn user_current(commit: Commit) -> Self { + let mm = kernel::current!().mm().map(|mm| ARef::from(&**mm)); + Self::user(commit, mm) + } + + fn pages(&self) -> isize { + isize::try_from(self.bytes / PAGE_SIZE as u64).unwrap_or(isize::MAX) + } + + fn apply(&self, held: bool) { + match &self.kind { + Kind::None => {} + Kind::Kernel(ledger) => { + if held { + ledger.kernel_resident.fetch_add(self.bytes, Ordering::AcqRel); + } else { + sub(&ledger.kernel_resident, self.bytes); + } + } + Kind::User { commit, mm } => { + if held { + commit.client.resident.fetch_add(self.bytes, Ordering::AcqRel); + commit + .ledger + .user_resident + .fetch_add(self.bytes, Ordering::AcqRel); + } else { + sub(&commit.client.resident, self.bytes); + sub(&commit.ledger.user_resident, self.bytes); + } + if let Some(mm) = mm { + let pages = self.pages(); + mm.add_rss_counter(RssCounter::Shmem, if held { pages } else { -pages }); + } + } + } + } + + /// Records that the GPU now holds (or no longer holds) the object's backing. + /// + /// Idempotent: only a change of state is counted. + pub(crate) fn set_held(&self, held: bool) { + if self.held.swap(held, Ordering::AcqRel) != held { + self.apply(held); + } + } +} + +impl Drop for Charge { + fn drop(&mut self) { + // Uncharge before `kind` (and with it the mm reference) is dropped. + self.set_held(false); + } +} + +#[kunit_tests(asahi_memacct)] +mod tests { + use super::*; + + const GIB: u64 = 1 << 30; + const MIB: u64 = 1 << 20; + + #[test] + fn default_limits_follow_ram() { + // The module parameter defaults. + let l = limits_for(8 * GIB, 67, 50); + assert_eq!(l.device, 8 * GIB * 67 / 100); + assert_eq!(l.client, 4 * GIB); + } + + #[test] + fn disabled_and_clamped_limits() { + assert_eq!(limits_for(8 * GIB, 0, 0), Limits { device: 0, client: 0 }); + assert_eq!(limits_for(8 * GIB, 100, 0), Limits { device: 0, client: 0 }); + // A client limit above the device limit is clamped to it. + let l = limits_for(8 * GIB, 25, 50); + assert_eq!(l.client, l.device); + // No client limit with a device limit means the device limit. + let l = limits_for(8 * GIB, 50, 0); + assert_eq!(l.client, 4 * GIB); + // No device limit leaves the client limit alone. + let l = limits_for(8 * GIB, 0, 25); + assert_eq!(l, Limits { device: 0, client: 2 * GIB }); + } + + #[test] + fn percentages_do_not_overflow() { + let l = limits_for(u64::MAX, 99, 98); + assert!(l.device > l.client); + assert!(l.device <= u64::MAX / 100 * 99 + 99); + } + + #[test] + fn try_add_respects_limit_and_extra() { + let c = AtomicU64::new(0); + assert!(try_add_within(&c, 6, 10, 0)); + assert!(!try_add_within(&c, 5, 10, 0)); + assert!(try_add_within(&c, 4, 10, 0)); + assert_eq!(c.load(Ordering::Relaxed), 10); + let d = AtomicU64::new(0); + assert!(!try_add_within(&d, 6, 10, 5)); + assert!(try_add_within(&d, 5, 10, 5)); + let e = AtomicU64::new(u64::MAX - 1); + assert!(!try_add_within(&e, 2, 0, 0)); + assert!(try_add_within(&e, 1, 0, 0)); + } + + static ADMIT: Ledger = Ledger::new(); + + #[test] + fn admission_reserves_and_releases() -> Result { + ADMIT.set_limits(Limits { device: 100 * MIB, client: 60 * MIB }); + let a = Client::new()?; + let b = Client::new()?; + let c = Client::new()?; + let a1 = ADMIT.admit(&a, 40 * MIB)?; + // The client limit refuses a's second 40 MiB; nothing stays reserved. + assert!(ADMIT.admit(&a, 40 * MIB).is_err()); + assert_eq!(a.committed(), 40 * MIB); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 40 * MIB); + let b1 = ADMIT.admit(&b, 50 * MIB)?; + // The device limit refuses c's 20 MiB and rolls back c's reservation. + assert!(ADMIT.admit(&c, 20 * MIB).is_err()); + assert_eq!(c.committed(), 0); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 90 * MIB); + assert_eq!(ADMIT.refused.load(Ordering::Relaxed), 2); + drop(a1); + assert_eq!(a.committed(), 0); + let c1 = ADMIT.admit(&c, 20 * MIB)?; + drop((b1, c1)); + assert_eq!(b.committed(), 0); + assert_eq!(c.committed(), 0); + assert_eq!(ADMIT.user_committed.load(Ordering::Relaxed), 0); + Ok(()) + } + + static KERNEL: Ledger = Ledger::new(); + + #[test] + fn kernel_objects_count_against_the_device_limit() -> Result { + KERNEL.set_limits(Limits { device: 64 * MIB, client: 64 * MIB }); + let k = Charge::kernel(&KERNEL, 48 * MIB); + k.set_held(true); + let a = Client::new()?; + assert!(KERNEL.admit(&a, 32 * MIB).is_err()); + let c = KERNEL.admit(&a, 16 * MIB)?; + drop(k); + assert_eq!(KERNEL.kernel_resident.load(Ordering::Relaxed), 0); + drop(c); + Ok(()) + } + + static HELD: Ledger = Ledger::new(); + + #[test] + fn held_transitions_are_idempotent() -> Result { + HELD.set_limits(Limits { device: 0, client: 0 }); + let a = Client::new()?; + let charge = Charge::user(HELD.admit(&a, 8 * MIB)?, None); + assert!(!charge.held.load(Ordering::Relaxed)); + charge.set_held(true); + charge.set_held(true); + assert_eq!(a.resident(), 8 * MIB); + assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 8 * MIB); + charge.set_held(false); + charge.set_held(false); + assert_eq!(a.resident(), 0); + charge.set_held(true); + // Dropping a held object uncharges it and returns its commit. + drop(charge); + assert_eq!(a.resident(), 0); + assert_eq!(a.committed(), 0); + assert_eq!(HELD.user_resident.load(Ordering::Relaxed), 0); + assert_eq!(HELD.user_committed.load(Ordering::Relaxed), 0); + Ok(()) + } + + #[test] + fn imports_are_not_counted() { + let charge = Charge::none(); + charge.set_held(true); + assert!(charge.held.load(Ordering::Relaxed)); + drop(charge); + } + + static MM: Ledger = Ledger::new(); + + // `mm_alloc()` is only exported to modules in the KUnit namespace. + #[cfg(not(MODULE))] + #[test] + fn held_pages_are_charged_to_the_creating_mm() -> Result { + MM.set_limits(Limits { device: 0, client: 0 }); + // SAFETY: `mm_alloc()` returns a new mm with one user and one count, or NULL. + let raw = unsafe { kernel::bindings::mm_alloc() }; + if raw.is_null() { + return Err(ENOMEM); + } + // SAFETY: `raw` is a live mm until the `mmput()` below. + let mm = ARef::::from(unsafe { Mm::from_raw(raw) }); + let a = Client::new()?; + let charge = Charge::user(MM.admit(&a, 4 * PAGE_SIZE as u64)?, Some(mm.clone())); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + charge.set_held(true); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 4); + assert_eq!(mm.rss_counter(RssCounter::Anon), 0); + charge.set_held(false); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + charge.set_held(true); + // Freeing a held object uncharges it before dropping its mm reference. + drop(charge); + assert_eq!(mm.rss_counter(RssCounter::Shmem), 0); + drop(mm); + // SAFETY: Drops the user reference `mm_alloc()` returned. `__mmdrop()` then + // checks that every RSS counter is back to zero. + unsafe { kernel::bindings::mmput(raw) }; + Ok(()) + } +} + +#[kernel::macros::kunit_tests(asahi_gem_memory_limits)] +mod exported_limits_tests { + use super::*; + + #[test] + fn exported_capacities_match_admission() { + let ledger = Ledger::new(); + assert_eq!(ledger.limits(), Limits { device: 0, client: 0 }); + let limits = limits_for(8 << 30, 67, 50); + ledger.set_limits(limits); + assert_eq!(ledger.limits(), limits); + assert!(limits.client < limits.device); + assert_eq!(limits.client, 4 << 30); + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 2bf4a6823024d..cda3ac8d206bd 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -15,7 +15,7 @@ //! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; -use core::mem::size_of; +use core::mem::{size_of, ManuallyDrop}; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ @@ -29,7 +29,6 @@ use core::sync::atomic::{ use kernel::{ addr::{ - DmaAddr, PhysicalAddr, // }, c_str, @@ -115,14 +114,15 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; -mod shared; mod lifetime; +mod ppl; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod m3; mod residency; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; +mod shared; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; @@ -156,23 +156,81 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// Two coprocessor instances that never touch the handoff region (G17). + /// One or two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, the secondary one the root at - /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel - /// context. + /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at + /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. + /// TTBAT context 1 starts as an alias of the kernel context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: u64, + secondary_root_offset: Option, }, } +/// UAT policy of a SoC where it differs from the A18 Pro's. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UatQuirks { + /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and + /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of + /// `struct page`. + /// + /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the + /// regions lose `no-map` and this path goes away. + pub(crate) reserved_tables: bool, + // The following settings keep a SoC on the only policy it was tested with. Each one is + // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. + /// Write back every page-table and context-table change: the GPU table walker is not + /// assumed to snoop the CPU caches. + pub(crate) clean_tables: bool, + /// Invalidate ranges one page at a time, never with range TLBI operations. + pub(crate) page_tlbi: bool, + /// Take the handoff lock around context-table edits. Where the firmware leaves its half of + /// the handoff page alone, its lock bytes are not initialized and are never taken. + pub(crate) handoff_lock: bool, + /// Handoff "current slot" written at initialization. + pub(crate) handoff_slot: u32, + /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in + /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. + pub(crate) native_gpu_permissions: bool, +} + +impl UatQuirks { + /// The A18 Pro and earlier policy. + pub(crate) const DEFAULT: UatQuirks = UatQuirks { + reserved_tables: false, + clean_tables: false, + page_tlbi: false, + handoff_lock: true, + handoff_slot: 0, + native_gpu_permissions: false, + }; +} + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; +/// M4's nonzero wire incarnation of a host-owned root installation. +const fn binding_wire_generation(incarnation: u64) -> u8 { + if incarnation == 0 { + 0 + } else { + (1 + (incarnation - 1) % 255) as u8 + } +} + +/// Upper word of an application TTBAT slot. M4 exposes firmware upper mappings only in +/// context 0; the existing G15 handoff shares them with user contexts. +const fn application_ttb1(root: u64, asid: usize, map_kernel_to_user: bool) -> u64 { + if map_kernel_to_user { + root | TTBR_VALID | (asid as u64) << TTBR_ASID_SHIFT + } else { + 0 + } +} + /// UAT address space geometry of a given SoC. #[derive(Debug, Copy, Clone)] pub(crate) struct UatGeometry { @@ -363,6 +421,8 @@ struct VmBinding { active_users: usize, binding: Option>, bind_token: Option, + /// Nonzero M4 wire incarnation of the root currently owning this slot. + generation: u8, ttb: u64, } @@ -375,7 +435,20 @@ struct VmBoData { /// linking the GPUVA. Remap pieces keep their original sequence. newest_bind: AtomicU64, #[pin] - segments: Mutex>>, + segments: Mutex>, +} + +/// Copied DMA addresses and the lease covering every alias of this VM/BO association. +/// The lease is dropped only after all aliases have been unmapped and invalidated. +struct SgSegments { + segments: KVVec<(DmaAddress, ResourceSize)>, + _lease: kernel::drm::gem::shmem::SGLease, +} + +impl SgSegments { + fn iter(&self) -> core::slice::Iter<'_, (DmaAddress, ResourceSize)> { + self.segments.iter() + } } /// Preallocated resources needed to execute a VM operation. @@ -494,22 +567,16 @@ impl gpuvm::DriverGpuVm for VmInner { let one_page = op.flags().contains(gpuvm::GpuVaFlag::REPEAT); - let Some(dev) = ctx.dev else { - dev_err!(self.dev.as_ref(), "StepContext missed Bound device!"); + if ctx.dev.is_none() { return Err(ENODEV); - }; - - let sgt = op.obj().sg_table(dev).inspect_err(|e| { - dev_err!( - self.dev.as_ref(), - "sm_step_map() Failed to get SG table: {:?}", - e - ); - })?; - for sgt_entry in sgt.iter() { - // TODO: proper DMA address/length handling - let mut addr = sgt_entry.dma_address() as usize; - let mut len: usize = sgt_entry.dma_len() as usize; + } + // Bind preparation retained the table's backing before taking the VM reservation. + // Never acquire the object reservation from inside the GPUVM map callback. + let segments = op.vm_bo().data().segments.lock(); + let sgt = segments.as_ref().ok_or(EFAULT)?; + for &(dma_address, dma_len) in sgt.iter() { + let mut addr = dma_address as usize; + let mut len = dma_len as usize; if left == 0 { break; @@ -556,6 +623,7 @@ impl gpuvm::DriverGpuVm for VmInner { iova += len as u64; } + drop(segments); let gpuva = match ctx.preallocated_gpuva() { Ok(gpuva) => gpuva, Err(error) => { @@ -641,7 +709,8 @@ impl gpuvm::DriverGpuVm for VmInner { let _mutation = self.mapping_mutation(); if orig_invalidated { - self.page_table.discard_partial_map(unmap_start..unmap_end)?; + self.page_table + .discard_partial_map(unmap_start..unmap_end)?; } else { self.page_table.unmap_pages(unmap_start..unmap_end)?; } @@ -684,7 +753,12 @@ impl gpuvm::DriverGpuVm for VmInner { impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); + mod_dev_dbg!( + self.dev, + "MMU: unmap: {:#x}:{:#x}\n", + va.addr(), + va.length() + ); let _mutation = self.mapping_mutation(); if va.flags().contains(GpuVaFlag::INVALIDATED) { @@ -719,14 +793,7 @@ impl VmInner { // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - mem::sync(); - self.tlbi_range(asid as u8, addr, size as usize); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size as usize); + self.finish_failed_map(addr, size as usize); if let Err(error) = result { dev_err!( self.dev.as_ref(), @@ -736,6 +803,19 @@ impl VmInner { } } + /// Complete discarded stores and invalidate both the legacy bound slot and + /// every M4/G17 context naming this root. Call only after releasing PPL. + fn finish_failed_map(&self, addr: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + mem::sync(); + self.tlbi_range(asid as u8, addr, size); + mem::sync(); + } + } + self.tlbi_contexts_naming_root(addr, size); + } + fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { @@ -790,15 +870,19 @@ impl VmInner { /// /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Other firmware only uses the bind slot, which the callers - /// already invalidate. + /// change must reach all of them. Adopted M4 roots also remain in the TTBAT after a bind + /// lease is dropped, until the slot is reassigned or the root is destroyed. Include those + /// contexts before allowing an unmapped object's backing to be released. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { + if self.uat_inner.firmware == UatFirmware::Handoff && !self.uat_inner.m4_adopted { mem::sync(); return; } - let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + let contexts = self + .uat_inner + .lock() + .contexts_naming_root(self.ttb(), self.uat_inner.m4_adopted); self.tlbi_context_mask(iova, size, contexts); } @@ -826,10 +910,35 @@ impl VmInner { .checked_add(node.mapped_size as u64) .ok_or(EOVERFLOW)?; let _mutation = self.mapping_mutation(); + // Allocate detached RAM tables before excluding the firmware. The shadow L2 is not + // the reserved shared-L2; only final leaf stores below already linked L3s are live. self.page_table.prepare_map(node.start()..end)?; + let uat_inner = self.uat_inner.clone(); + let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; let result = self.map_node_prepared(node, prot); + let discard = if result.is_err() { + // Rollback still changes live leaves and must remain under the PPL lease. + self.page_table.discard_partial_map(node.start()..end) + } else { + Ok(()) + }; + // The M4 handoff guard owns UatShared's mutex. Invalidation follows its release. + drop(handoff); if result.is_err() { - self.rollback_failed_map(node.start(), node.mapped_size as u64); + self.finish_failed_map(node.start(), node.mapped_size); + } else { + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + } + if let Err(error) = discard { + if self.uat_inner.m4_adopted { + // The caller retains its node and backing when this latch is set. + self.uat_inner.failed.store(true, Ordering::Release); + } + dev_err!( + self.dev.as_ref(), + "Could not discard failed kernel mapping: {:?}\n", + error + ); } result } @@ -837,7 +946,14 @@ impl VmInner { /// Install only into the empty range reserved by `map_node`. fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); + let segments = node + .bo + .as_ref() + .ok_or(EINVAL)? + .get()? + .data() + .segments + .lock(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -895,8 +1011,7 @@ impl VmInner { } } - // A context naming this root may be live and hold a negative translation for the range. - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + // The caller invalidates negative translations after releasing its PPL lease. Ok(()) } } @@ -974,7 +1089,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -982,8 +1097,9 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); } - inner.handoff().unlock(); + inner.handoff_unlock(); owned || (low == 0 && high == 0) }; if release { @@ -1024,6 +1140,12 @@ impl VmBind { pub(crate) fn slot(&self) -> u32 { self.1 } + + /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn generation(&self) -> u8 { + self.0.binding.lock().generation + } } impl Drop for VmBind { @@ -1064,10 +1186,7 @@ struct MappingBo { owner: ARef>, } impl MappingBo { - fn new( - bo: ARef>, - owner: ARef>, - ) -> Self { + fn new(bo: ARef>, owner: ARef>) -> Self { Self { bo: Some(bo), owner, @@ -1117,22 +1236,51 @@ pub(crate) struct TimestampMapping { } impl TimestampMapping { + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + Ok(Self { + mapping: gem.owned_vmap::<0>()?, + offset, + }) + } + pub(crate) fn write(&self, value: u64) { + // Completion runs in task context. Bindings of the same BO share this lock, + // so aligned u64 and overlapping unaligned byte stores never race in Rust. + let _guard = self.mapping.owner().timestamp_lock.lock(); + // SAFETY: timestamp_mapping checked the complete extent inside this pinned vmap. let base = self.mapping.addr.cast::(); - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { - &*base.add(self.offset + index).cast::() - }; - destination.store(byte, Ordering::Relaxed); + let pointer = unsafe { base.add(self.offset) }; + if pointer as usize % core::mem::align_of::() == 0 { + // SAFETY: timestamp_mapping checked all eight bytes; the vmap pins the backing. + // A naturally aligned query result becomes available in one complete store. + let destination = unsafe { &*pointer.cast::() }; + destination.store(value.to_le(), Ordering::Relaxed); + } else { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: timestamp_mapping checked the complete eight-byte extent. + // The owned vmap pins its backing; byte atomics permit unaligned and + // overlapping timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(self.offset + index).cast::() }; + destination.store(byte, Ordering::Relaxed); + } } + // A CPU-retired query can be consumed by a later GPU command. WRITEBACK + // objects use a cacheable vmap, while GPU mappings need not snoop that cache. + // Publish the bytes before signalling the completion fence or queue frontier. + if self.mapping.owner().is_writeback() { + crate::pgtable::clean_poc(pointer, size_of::()); + } + crate::pgtable::poc_barrier(); } } /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); +pub(crate) struct KernelMapping(ManuallyDrop>); impl KernelMapping { /// Returns the IOVA base of this mapping @@ -1156,12 +1304,14 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) + lock_vm(&self.0.owner, None, false)? + .page_table + .translate_iova(address) } - /// Writes a userspace timestamp through the retained GEM mapping. Byte - /// stores support unaligned destinations and concurrent completion writes - /// without creating mutable aliases. The complete timestamp is not atomic. + /// Writes a userspace timestamp through the retained GEM mapping. Aligned results + /// use a complete atomic store; byte stores support unaligned destinations without + /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; @@ -1174,10 +1324,7 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - Ok(TimestampMapping { - mapping: gem.owned_vmap::<0>()?, - offset: start, - }) + TimestampMapping::new(gem, start) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1292,10 +1439,10 @@ impl KernelMapping { let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff().lock(); + uat_inner.handoff_lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff().unlock(); + uat_inner.handoff_unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1385,6 +1532,19 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { + // An ASC-cached M4 mapping has no qualified cache-flush teardown. Even an unexpected + // failed construction must retain its complete node/backing instead of invoking the + // unrelated G13/G14 FwCtl protocol. + if self.0.uat_inner.m4_adopted && self.0.prot.is_cached_noncoherent() { + self.0.uat_inner.failed.store(true, Ordering::Release); + pr_err!( + "MMU: retaining cached M4 mapping {:#x}+{:#x} until reboot\n", + self.iova(), + self.size() + ); + return; + } + // This is the main unmap function for UAT mappings. // The sequence of operations here is finicky, due to the interaction // between cached GFX ASC mappings and the page tables. These mappings @@ -1427,8 +1587,27 @@ impl Drop for KernelMapping { self.size() ); + let upper_handoff = match self.0.uat_inner.lock_m4_upper(owner.is_kernel) { + Ok(guard) => guard, + Err(error) => { + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after PPL failure: {:?}\n", + error + ); + return; + } + }; let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { + if self.0.uat_inner.m4_adopted { + self.0.uat_inner.failed.store(true, Ordering::Release); + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after partial unmap failure\n" + ); + return; + } dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -1437,6 +1616,8 @@ impl Drop for KernelMapping { ); } + drop(upper_handoff); + if let Some(asid) = owner.slot() { mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); @@ -1453,6 +1634,11 @@ impl Drop for KernelMapping { // The range returns to the VA allocator once this mapping is gone, so no context that // names this root may keep a stale translation for it. owner.tlbi_contexts_naming_root(self.iova(), self.size()); + drop(_mutation); + drop(owner); + // SAFETY: Every normal path reaches this exactly once, after all GPUVM locks have + // been released. Uncertain M4 paths return earlier, retaining the node and its pins. + unsafe { ManuallyDrop::drop(&mut self.0) }; } } @@ -1461,14 +1647,44 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], + /// Firmware-observed incarnations of application roots installed by the VM slot allocator. + binding_generations: [u64; UAT_NUM_CTX], + binding_published: [bool; UAT_NUM_CTX], + /// Expected lower words of user roots owned by the VM slot allocator. Replaced only by + /// successful host publications, never learned by accepting observed hardware words. + owned_user_roots: [u64; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, + quirks: UatQuirks, } impl UatShared { + /// Takes the handoff lock, where this SoC uses it. + fn handoff_lock(&self) { + if self.quirks.handoff_lock { + self.handoff().lock(); + } + } + + /// Releases the handoff lock taken by [`Self::handoff_lock`]. + fn handoff_unlock(&self) { + if self.quirks.handoff_lock { + self.handoff().unlock(); + } + } + + /// Writes back context-table slot `idx` after it changed, where this SoC needs it. + fn publish_slot(&self, idx: usize) { + if self.quirks.clean_tables { + let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + pgtable::clean_poc(slot, size_of::()); + pgtable::poc_barrier(); + } + } + /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant @@ -1482,11 +1698,15 @@ impl UatShared { } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64) -> u64 { + fn contexts_naming_root(&self, ttb: u64, adopted: bool) -> u64 { self.ttbs() .iter() .enumerate() - .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) + .filter(|(ctx, slot)| { + let expected = tagged_root(ttb, *ctx); + let _ = adopted; // Context 0 is naturally untagged; user contexts carry their ASID. + slot.ttb0.load(Ordering::Acquire) == expected + }) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } } @@ -1503,20 +1723,138 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// The stopped J713 handoff: bounded full-barrier PPL and upper words zero in users. + m4_adopted: bool, + /// An uncertain mapping/root is retained; future M4 allocation and publication fail. + failed: AtomicBool, + /// Clean page tables to the point of coherency for a walker that does not snoop the CPU + /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. clean_poc: bool, + /// This SoC's UAT policy. + quirks: UatQuirks, + /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). + reserved: Option>, #[pin] shared: Mutex, #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } +/// A transport owns the sole host-side PPL mutex before touching AP interest. +struct M4Transport<'a> { + shared: Guard<'a, UatShared, MutexBackend>, + start: Instant, + failed: &'a AtomicBool, +} + +impl ppl::Memory for M4Transport<'_> { + fn firmware_interested(&self) -> bool { + self.shared.handoff().lock_fw.load(Ordering::Relaxed) != 0 + } + fn firmware_turn(&self) -> bool { + self.shared.handoff().turn.load(Ordering::Relaxed) != 0 + } + fn ap_interest(&self, interested: bool) { + self.shared + .handoff() + .lock_ap + .store(u8::from(interested), Ordering::Relaxed); + } + fn give_firmware_turn(&self) { + self.shared.handoff().turn.store(1, Ordering::Relaxed); + } + fn barrier(&self) { + mem::sync(); + } + fn expired(&self) -> bool { + self.start.elapsed() > Delta::from_millis(100) + } + fn relax(&self) { + core::hint::spin_loop(); + } + fn fail(&self) { + self.failed.store(true, Ordering::Release); + } +} + +/// Every AP firmware transaction owns the same mutex, including G16 Window operations. +/// M4 drops its protocol guard before its transport releases the host mutex. The legacy +/// handoff path keeps the preexisting protocol and lock ordering for other generations. +#[must_use = "retain the guard until protected table mutations have completed"] +enum HandoffGuard<'a> { + M4(ppl::Guard>), + Legacy(Guard<'a, UatShared, MutexBackend>), +} + +impl HandoffGuard<'_> { + fn shared_mut(&mut self) -> &mut UatShared { + match self { + Self::M4(guard) => &mut guard.memory_mut().shared, + Self::Legacy(shared) => shared, + } + } +} + +impl Drop for HandoffGuard<'_> { + fn drop(&mut self) { + if let Self::Legacy(shared) = self { + shared.handoff_unlock(); + } + } +} + +/// Opaque host-and-firmware lease for G16 window transactions. No caller may +/// independently manipulate the AP-interest or turn words. +#[cfg(CONFIG_DRM_ASAHI_G16)] +#[must_use = "retain the guard throughout the reserved-table window transaction"] +pub(crate) struct M4HandoffGuard<'a> { + _transaction: HandoffGuard<'a>, +} + impl UatInner { /// Take the lock on the shared data and return the guard. fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } + fn check_m4(&self) -> Result { + if self.m4_adopted && self.failed.load(Ordering::Acquire) { + Err(EIO) + } else { + Ok(()) + } + } + + /// Serialize AP callers before taking the firmware exclusion. Timeouts withdraw + /// AP interest while still holding the mutex and latch M4 failure before releasing it. + fn lock_handoff(&self) -> Result> { + self.check_m4()?; + let shared = self.lock(); + self.check_m4()?; + if !self.m4_adopted { + shared.handoff_lock(); + return Ok(HandoffGuard::Legacy(shared)); + } + let transport = M4Transport { + shared, + start: Instant::::now(), + failed: &self.failed, + }; + match ppl::Guard::acquire(transport) { + Ok(guard) => Ok(HandoffGuard::M4(guard)), + Err(()) => Err(ETIMEDOUT), + } + } + + /// No leaf is changed on a failed acquisition. Uncertain mappings retain their backing. + fn lock_m4_upper(&self, upper: bool) -> Result>> { + if upper && self.m4_adopted { + self.lock_handoff().map(Some) + } else { + Ok(None) + } + } + /// Take a lock on a handoff flush slot and return the guard. fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() @@ -1569,9 +1907,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(0, Ordering::Relaxed); + self.cur_slot.store(slot, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -1654,16 +1992,36 @@ impl HandoffFlush { fn prefetch_sgt_segments( dev: &device::Device, gem: &gem::Object, -) -> Result> { +) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; - // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(segments) + Ok(SgSegments { segments, _lease: sgt.into_lease() }) +} + +/// Reuse cached segments without allocating another copy for each bind. The bind lease was +/// acquired before BO locking: its getter synchronized with any earlier reclaim under resv, +/// and its extra count excludes new eviction until preparation/commit finishes. +fn prepare_sgt_segments( + dev: &device::Device, + gem: &gem::Object, + bo: &gpuvm::GpuVmBo, + _bind_lease: &kernel::drm::gem::shmem::SGLease, +) -> Result { + let needs_segments = bo.data().segments.lock().is_none(); + if needs_segments { + // Never enter the object reservation while holding the BO mutex. + let prefetched = prefetch_sgt_segments(dev, gem)?; + let mut segments = bo.data().segments.lock(); + if segments.is_none() { + segments.replace(prefetched); + } + } + Ok(()) } impl Vm { @@ -1686,12 +2044,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, + uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? + .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; - - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -1717,6 +2075,7 @@ impl Vm { VmBinding { binding: None, bind_token: None, + generation: 0, active_users: 0, ttb: page_table.ttb(), }, @@ -1771,7 +2130,10 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); @@ -1784,13 +2146,16 @@ impl Vm { /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn with_tracked_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); Ok(self) } - /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it; G17 timeout + /// attribution compares and logs VMs by it). pub(crate) fn id(&self) -> u64 { self.id } @@ -1899,9 +2264,7 @@ impl Vm { return false; }; Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked( - &mut inner, guard, stride - block_size, false, false, - ) + && !Self::covers_range_locked(&mut inner, guard, stride - block_size, false, false) }) } @@ -1962,6 +2325,37 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + ) + } + + /// Lowest-address placement for a monotonic search across several VMs. + pub(crate) fn map_in_range_lowest( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + ) + } + + #[allow(clippy::too_many_arguments)] + fn map_in_range_with_mode( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -1976,17 +2370,16 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let size = object_range.range(); let mut vm_exec = unsafe { @@ -1994,14 +2387,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM @@ -2022,7 +2407,7 @@ impl Vm { 0, range.start, range.end, - mm::InsertMode::Best, + mode, ); let mut node = match result { Ok(node) => node, @@ -2040,9 +2425,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a GEM object into this Vm at a specific address. @@ -2060,17 +2452,16 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(&gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { @@ -2082,14 +2473,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // Keep reservation-locking destructors out of the allocator payload @@ -2124,9 +2507,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a range of a GEM object into this Vm using GPUVM. @@ -2144,17 +2534,16 @@ impl Vm { let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev = unsafe { self.dev.as_bound() }; + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let mut resources = VmOpResources { preallocated_gpuvas: [ @@ -2172,14 +2561,6 @@ impl Vm { }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - // Preallocate the page tables, to fail early if we ENOMEM inner.page_table.alloc_pages(addr..(addr + size))?; @@ -2222,6 +2603,22 @@ impl Vm { vm_exec.unique_gpuvm.sm_map(req) } + /// Spend the binding generation only for an incarnation exposed to firmware. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn mark_binding_published(&self) -> Result { + let uat = self.gpuvm.lock_inner().uat_inner.clone(); + let binding = self.binding.lock(); + let slot = binding.binding.as_ref().ok_or(EIO)?.slot() as usize + UAT_USER_CTX_START; + let mut shared = uat.lock(); + if shared.owned_user_roots[slot] != tagged_root(binding.ttb, slot) + || shared.ttbs()[slot].ttb0.load(Ordering::Acquire) != shared.owned_user_roots[slot] + { + return Err(EIO); + } + shared.binding_published[slot] = true; + Ok(()) + } + /// Add a direct MMIO mapping to this Vm at a free address. pub(crate) fn map_io( &self, @@ -2265,22 +2662,35 @@ impl Vm { let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; + let transaction_uat = inner.uat_inner.clone(); + let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); - let ret = inner.page_table.map_pages( - iova..end, - phys as PhysicalAddr, - prot, - false, - ); + let ret = inner + .page_table + .map_pages(iova..end, phys as PhysicalAddr, prot, false); + let discard = if ret.is_err() { + inner.page_table.discard_partial_map(iova..end) + } else { + Ok(()) + }; + drop(handoff); if ret.is_err() { - inner.rollback_failed_map(iova, size as u64); + inner.finish_failed_map(iova, size); + } else { + inner.tlbi_contexts_naming_root(iova, size); + } + let retain = discard.is_err() && inner.uat_inner.m4_adopted; + if retain { + inner.uat_inner.failed.store(true, Ordering::Release); } - // Drop the exec_lock first, so that if map_node failed the - // KernelMappingInner destructur does not deadlock. drop(mutation); - let _ = inner; + drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Unmap everything in an address range. @@ -2485,8 +2895,18 @@ impl Drop for VmInner { let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.uat_inner.lock(); - uat_inner.handoff().lock(); + let mut transaction = match self.uat_inner.lock_handoff() { + Ok(guard) => guard, + Err(_) => { + self.page_table.retain(); + pr_err!( + "MMU: retaining M4 VM {} page tables after PPL failure\n", + self.id + ); + return; + } + }; + let uat_inner = transaction.shared_mut(); let handoff_cur = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); let inval = ttb_cur == ttb; @@ -2499,9 +2919,10 @@ impl Drop for VmInner { } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); + uat_inner.owned_user_roots[idx] = 0; + uat_inner.publish_slot(idx); } - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + drop(transaction); // In principle we dropped all the KernelMappings already, but we might as // well play it safe and invalidate the whole ASID. @@ -2532,6 +2953,18 @@ pub(crate) struct AdoptedRegions { } impl Uat { + /// Common AP and firmware exclusion for G16's reserved-table window. Acquire + /// immutable root addresses before this guard, preserving binding -> UAT lock order. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn lock_m4_handoff(&self) -> Result> { + if !self.inner.m4_adopted { + return Err(EINVAL); + } + Ok(M4HandoffGuard { + _transaction: self.inner.lock_handoff()?, + }) + } + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] /// Map a region given by its physical range, write-combined. fn map_physical(range: &Range, min_size: usize) -> Result { @@ -2581,10 +3014,24 @@ impl Uat { /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after /// mapping and before the firmware uses the new addresses. Page tables are never freed on /// unmap, so linked entries stay valid. + #[allow(dead_code)] pub(crate) fn new_adopted( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, + ) -> Result { + Self::new_adopted_with_user_upper(dev, geometry, regions, true) + } + + /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also + /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; + /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. + #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + pub(crate) fn new_adopted_with_user_upper( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + map_kernel_to_user: bool, ) -> Result { let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; @@ -2596,8 +3043,12 @@ impl Uat { let inner = Arc::pin_init( try_pin_init!(UatInner { firmware: UatFirmware::Handoff, + m4_adopted: !map_kernel_to_user, + failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, + reserved: None, + quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -2606,10 +3057,14 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, - map_kernel_to_user: true, + map_kernel_to_user, handoff_rgn, ttbs_rgn, + quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -2621,8 +3076,14 @@ impl Uat { let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let shadow_root = kernel::page::Page::into_phys(shadow); - let kernel_lower_vm = - Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_lower_vm = Vm::new( + dev.into(), + inner.clone(), + geometry.user_range(), + geometry, + None, + 1, + )?; let kernel_vm = Vm::new( dev.into(), inner.clone(), @@ -2725,7 +3186,10 @@ impl Uat { } owner.ttb() }; - Ok(self.inner.lock().contexts_naming_root(root)) + Ok(self + .inner + .lock() + .contexts_naming_root(root, self.inner.m4_adopted)) } /// Publish a completed alias set before its descriptors become visible. @@ -2754,7 +3218,9 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.inner.check_m4()?; let mut binding = vm.binding.lock(); + self.inner.check_m4()?; if binding.binding.is_none() { assert_eq!(binding.active_users, 0); @@ -2773,16 +3239,24 @@ impl Uat { let idx = (slot.slot() as usize) + UAT_USER_CTX_START; let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.inner.lock(); - - let ttb1 = if uat_inner.map_kernel_to_user { - uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT + let mut transaction = self.inner.lock_handoff()?; + let uat_inner = transaction.shared_mut(); + // On M4, empty bind/close churn must not wrap the wire generation + // back to firmware's last observed identity for a different root. + let previous = uat_inner.binding_generations[idx]; + let incarnation = if !self.inner.m4_adopted + || previous == 0 + || uat_inner.binding_published[idx] + { + previous.checked_add(1).ok_or(EOVERFLOW)? } else { - 0 + previous }; + let ttb1 = + application_ttb1(uat_inner.kernel_ttb1, idx, uat_inner.map_kernel_to_user); + let ttbs = uat_inner.ttbs(); - uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { pr_err!( "Vm::bind to slot {}, but it is currently in use by the ASC?\n", @@ -2791,8 +3265,12 @@ impl Uat { } ttbs[idx].ttb0.store(ttb, Ordering::Release); ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + uat_inner.binding_generations[idx] = incarnation; + uat_inner.binding_published[idx] = false; + uat_inner.owned_user_roots[idx] = ttb; + binding.generation = binding_wire_generation(incarnation); + uat_inner.publish_slot(idx); + drop(transaction); // Make sure all TLB entries from the previous owner of this ASID are gone mem::sync(); @@ -2811,6 +3289,35 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } + /// Confirms the exact stopped-adopted M4 TTBAT contract. The same mutex serializes every + /// host root installation/removal; firmware does not write these words. An unexpected + /// observed root is an error, never a new ownership grant. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn check_adopted_bindings(&self) -> Result { + self.inner.check_m4()?; + // Binding locks precede the global UAT lock, as in bind(). + let lower = self.kernel_lower_vm.ttb() | TTBR_VALID; + let inner = self.inner.lock(); + self.inner.check_m4()?; + if self.inner.firmware != UatFirmware::Handoff || inner.map_kernel_to_user { + return Err(EINVAL); + } + let ttbs = inner.ttbs(); + if ttbs[0].ttb0.load(Ordering::Acquire) != lower + || ttbs[0].ttb1.load(Ordering::Acquire) != inner.kernel_ttb1 | TTBR_VALID + { + return Err(EIO); + } + for index in UAT_USER_CTX_START..UAT_NUM_CTX { + if ttbs[index].ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || ttbs[index].ttb1.load(Ordering::Acquire) != 0 + { + return Err(EIO); + } + } + Ok(()) + } + /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { @@ -2862,6 +3369,7 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -2871,14 +3379,17 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - Vm::new( + self.inner.check_m4()?; + let vm = Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, self.geometry, None, id, - ) + )?; + self.inner.check_m4()?; + Ok(vm) } /// Returns the address space geometry of this UAT. @@ -2900,7 +3411,7 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { for (value, pte) in snapshot.iter_mut().zip(ptes) { *value = pte.load(Ordering::Relaxed); } @@ -2920,7 +3431,7 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { ptes.iter() .zip(snapshot) .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) @@ -2936,15 +3447,19 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let Some(secondary) = self.inner.lock().secondary_ttb1 else { + // A single-instance firmware has no secondary root to mirror. + return Ok(()); + }; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(secondary, count, |secondary| { + pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(reserved, secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -2961,24 +3476,33 @@ impl Uat { Ok(()) } + /// Discards the CPU's cached lines of the no-map table regions after the firmware built its + /// root there during boot, before the first host edit (`UatQuirks::reserved_tables`). + pub(crate) fn invalidate_reserved_tables(&self) { + if let Some(reserved) = self.inner.reserved.as_ref() { + reserved.invalidate(); + } + } + /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff().unlock(); + inner.handoff_unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); + inner.publish_slot(KERNEL_ALIAS_CTX); + inner.handoff_unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -2988,7 +3512,12 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { + fn make_inner( + dev: driver::AsahiDevice, + firmware: UatFirmware, + quirks: UatQuirks, + reserved: Option>, + ) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; @@ -2997,12 +3526,16 @@ impl Uat { let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] - clean_poc: false, + m4_adopted: false, + failed: AtomicBool::new(false), + clean_poc: quirks.clean_tables, + quirks, + reserved, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3011,10 +3544,14 @@ impl Uat { kernel_ttb1: 0, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, + quirks: shared_quirks, }, "uat_shared" ), @@ -3030,12 +3567,27 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, + quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - let inner = Self::make_inner(dev.clone(), firmware)?; - + mem::set_page_tlbi(quirks.page_tlbi); let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let reserved = if quirks.reserved_tables { + let mut tables = pgtable::ReservedTables::new(); + for name in [c"pagetables", c"l2-pagetables"] { + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: The boot loader reserved these regions (no-map) for the firmware's + // page tables; they stay reserved until reboot. + unsafe { tables.add(res)? }; + } + Some(Arc::new(tables, GFP_KERNEL)?) + } else { + None + }; + + let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -3043,11 +3595,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset, - } => ( - secondary_root_offset as usize + UAT_PGSZ, - Some(ttb1 + secondary_root_offset), - ), + secondary_root_offset: Some(offset), + } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), + UatFirmware::PublishedRoots { + secondary_root_offset: None, + } => (UAT_PGSZ, None), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -3098,11 +3650,15 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware)?; + inner.handoff().init(firmware, quirks.handoff_slot)?; + if quirks.clean_tables { + pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::poc_barrier(); + } dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); @@ -3114,7 +3670,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if secondary_ttb1.is_some() { + if matches!(firmware, UatFirmware::PublishedRoots { .. }) { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3123,8 +3679,12 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } + if quirks.clean_tables { + pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::poc_barrier(); + } - inner.handoff().unlock(); + inner.handoff_unlock(); core::mem::drop(inner); @@ -3142,3 +3702,30 @@ impl Drop for Uat { mem::sync(); } } + +impl Vm { + /// Diagnostic reservation occupancy (including guards), not a placement promise. + /// Removal can progress between queries; allocations are excluded by the VM lock. + pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { + let inner = lock_vm(&self.gpuvm, None, false)?; + let mut next = range.start; + let mut used = 0; + let mut largest = 0; + while next < range.end { + let Some(node) = inner.mm.first_overlap(next..range.end)? else { + largest = largest.max(range.end - next); + break; + }; + largest = largest.max(node.start.saturating_sub(next)); + let end = node.end.min(range.end); + used += end - node.start.max(next); + next = end; + } + Ok((used, largest)) + } +} +impl Uat { + pub(crate) fn execution_contexts_used(&self) -> u32 { + (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() + } +} diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 44a1575ecfd2b..e222c7cd8e6a9 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -17,6 +17,7 @@ pub(crate) struct PreparedUserMap { single_page: bool, context: context::PreparedBinding, _residency: Option, + _sg_lease: kernel::drm::gem::shmem::SGLease, } pub(crate) struct PreparedUserUnmap { @@ -57,7 +58,7 @@ impl PreparedUserBindBatch { } impl Vm { - /// Prepares host state without taking DMA reservations or changing any leaf. + /// Prepares host state before taking the GPUVM exec lock or changing any leaf. pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, @@ -80,24 +81,20 @@ impl Vm { // SAFETY: As in `Vm::bind_object()`, the device is bound while its file can issue binds. let dev = unsafe { self.dev.as_bound() }; // Populate the scatter-gather table before any GPUVM lock is taken. - crate::gem::held_sg_table(gem, dev)?; + let sg_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm .obtain(gem, pin_init!(VmBoData { newest_bind: AtomicU64::new(0), segments <- new_mutex!(None), }))?; - { - let mut segments = vm_bo.data().segments.lock(); - if segments.is_none() { - segments.replace(prefetch_sgt_segments(dev, gem)?); - } - } + prepare_sgt_segments(dev, gem, &vm_bo, &sg_lease)?; if (addr | size | offset) & UAT_PGMSK as u64 != 0 { return Err(EINVAL); } Ok(PreparedUserMap { _residency: residency, + _sg_lease: sg_lease, gem: gem.into(), vm_bo, resources, diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 30010be497973..66d507c2d7694 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -79,9 +79,13 @@ pub(super) struct VmLifetime { impl VmLifetime { pub(super) fn can_reclaim(&self) -> bool { self.state.try_lock().is_some_and(|state| { - state.active == 0 && state.commits == 0 - && !state.blocked && !state.quarantined && !state.draining - && !state.closed && !state.cleanup_failed + state.active == 0 + && state.commits == 0 + && !state.blocked + && !state.quarantined + && !state.draining + && !state.closed + && !state.cleanup_failed }) } @@ -339,10 +343,12 @@ impl Vm { drop(previous); } - /// Whether userspace has permanently closed this VM's map admission. - #[cfg(CONFIG_DRM_ASAHI_G15)] + /// Whether the client's handle of this VM is gone ([`Self::unmap_user_ranges`] closed + /// admission for good). G15 releases its per-VM state after that. pub(crate) fn is_closed(&self) -> bool { - self.lifetime.as_ref().is_some_and(|lifetime| lifetime.state.lock().closed) + self.lifetime + .as_ref() + .is_some_and(|lifetime| lifetime.state.lock().closed) } /// Tests mappings and reservations which userspace must never replace. @@ -411,7 +417,10 @@ impl Vm { } if !state.draining { state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(VmJobGuard { vm: self.clone(), _residency: residency }); + return Ok(VmJobGuard { + vm: self.clone(), + _residency: residency, + }); } lifetime.changed.wait(&mut state); } @@ -471,7 +480,11 @@ impl Vm { Ok(()) } - pub(super) fn defer_bind_batch(&self, batch: &mut PreparedUserBindBatch, start: usize) -> Result { + pub(super) fn defer_bind_batch( + &self, + batch: &mut PreparedUserBindBatch, + start: usize, + ) -> Result { use super::bind::PreparedUserBindOp; let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); @@ -485,7 +498,8 @@ impl Vm { } if batch .ops - .iter().skip(start) + .iter() + .skip(start) .any(|op| matches!(op, PreparedUserBindOp::Map(_))) { return Err(EIO); @@ -496,16 +510,21 @@ impl Vm { // Reserve before accepting any range; a partial deferred batch must // not leave canonical metadata ahead of the accepted cleanup. state.unmaps.reserve(batch.ops.len() - start, GFP_KERNEL)?; - let retired = self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { - PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), - PreparedUserBindOp::Map(_) => None, - })); + let retired = + self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); // Transfer the preallocated split nodes along with each accepted range. // Already-committed prefix operations must not be replayed: their nodes // may have been consumed and another binding may now occupy the range. for op in batch.ops.drain(start..) { if let PreparedUserBindOp::Unmap(unmap) = op { - if !state.unmaps.iter().any(|old| old.iova == unmap.iova && old.size == unmap.size) { + if !state + .unmaps + .iter() + .any(|old| old.iova == unmap.iova && old.size == unmap.size) + { state.unmaps.push(unmap, GFP_KERNEL)?; } } @@ -530,16 +549,30 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } - if state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + if state + .unmaps + .iter() + .any(|old| old.iova == iova && old.size == size) + { return Ok(true); } drop(state); let unmap = self.prepare_user_unmap(iova, size)?; let mut state = lifetime.state.lock(); - if state.closed { return Err(ENOENT); } - if !state.blocked { return Ok(false); } - if !state.quarantined { return Err(EBUSY); } - if !state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + if state.closed { + return Err(ENOENT); + } + if !state.blocked { + return Ok(false); + } + if !state.quarantined { + return Err(EBUSY); + } + if !state + .unmaps + .iter() + .any(|old| old.iova == iova && old.size == size) + { state.unmaps.push(unmap, GFP_KERNEL)?; } Ok(true) @@ -634,8 +667,8 @@ impl Vm { let mut failure = None; loop { let close = state.close_ranges.take(); - let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); - let objects = core::mem::replace(&mut state.objects, KVec::new()); + let unmaps = core::mem::take(&mut state.unmaps); + let objects = core::mem::take(&mut state.objects); drop(state); if let Some((user, kernel)) = close { if let Err(error) = self.unmap_user_ranges_now(user, kernel) { @@ -671,7 +704,9 @@ impl Vm { // terminal failure instead of silently discarding its outcome. state.blocked = true; state.cleanup_failed = true; - if let Some(status) = &self.status { status.record(error); } + if let Some(status) = &self.status { + status.record(error); + } } else if !state.closed && !state.cleanup_failed { state.blocked = false; } diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs index a63101a813426..8611221af492a 100644 --- a/drivers/gpu/drm/asahi/mmu/m3.rs +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -93,7 +93,7 @@ impl Vm { drop(mutation); let _ = inner; ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// [`Vm::map_io`] at `iova`, without the per-mapping log line. diff --git a/drivers/gpu/drm/asahi/mmu/ppl.rs b/drivers/gpu/drm/asahi/mmu/ppl.rs new file mode 100644 index 0000000000000..7cdad5e1f3831 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/ppl.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The M4 two-party PPL protocol. The transport owns the host mutex for the entire +//! guard lifetime: firmware exclusion alone does not serialize two AP threads. +//! This core has no kernel dependencies so the production guard can be exercised +//! against deterministic firmware and concurrent host transports. + +pub(super) trait Memory { + fn firmware_interested(&self) -> bool; + fn firmware_turn(&self) -> bool; + fn ap_interest(&self, interested: bool); + fn give_firmware_turn(&self); + fn barrier(&self); + fn expired(&self) -> bool; + fn relax(&self); + /// Latch failure before the transport releases host exclusion on timeout. + fn fail(&self); +} + +/// Owns both the host-side lock (in `memory`) and successful firmware exclusion. +#[must_use = "retain the guard until every protected table mutation has completed"] +pub(super) struct Guard { + memory: T, +} + +impl Guard { + pub(super) fn acquire(memory: T) -> Result { + let timeout = || { + memory.ap_interest(false); + memory.barrier(); + memory.fail(); + Err(()) + }; + memory.ap_interest(true); + memory.barrier(); + while memory.firmware_interested() { + if memory.expired() { + return timeout(); + } + if memory.firmware_turn() { + memory.ap_interest(false); + memory.barrier(); + while memory.firmware_turn() { + if memory.expired() { + return timeout(); + } + memory.relax(); + } + memory.ap_interest(true); + memory.barrier(); + } + memory.relax(); + } + memory.barrier(); + Ok(Self { memory }) + } + + pub(super) fn memory_mut(&mut self) -> &mut T { + &mut self.memory + } +} + +impl Drop for Guard { + fn drop(&mut self) { + self.memory.give_firmware_turn(); + self.memory.barrier(); + self.memory.ap_interest(false); + self.memory.barrier(); + // Only after these stores complete may `memory` release the host mutex. + } +} diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 5370b87ccf1b5..49c2241bc399f 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -251,6 +251,8 @@ pub(crate) struct VmShrinker { // SAFETY: shrinker core serializes registration lifetime; callback VM operations // use its residency/reservation locks. raw is never dereferenced after free. unsafe impl Send for VmShrinker {} +// SAFETY: Shared callbacks use the VM's residency/reservation locks. The +// registration remains live until shrinker_free has drained all callbacks. unsafe impl Sync for VmShrinker {} impl VmShrinker { @@ -273,7 +275,7 @@ impl VmShrinker { unsafe { (*raw).count_objects = Some(Self::count); (*raw).scan_objects = Some(Self::scan); - (*raw).private_data = (&*owner as *const Self).cast_mut().cast(); + (*raw).private_data = core::ptr::from_ref(&*owner).cast_mut().cast(); (*raw).seeks = 2; (*raw).batch = 256; kernel::bindings::shrinker_register(raw); @@ -281,23 +283,36 @@ impl VmShrinker { Ok(owner) } + /// Count reclaimable pages. + /// + /// # Safety + /// + /// `raw` must be this owner's live registered shrinker. The core must + /// retain its private data until the callback returns. unsafe extern "C" fn count( raw: *mut kernel::bindings::shrinker, _sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: The registration owns this stable allocation until callbacks drain. let owner = unsafe { &*((*raw).private_data as *const Self) }; - owner.vm.idle_reclaim_count() as _ + owner.vm.idle_reclaim_count() } + /// Reclaim idle pages. + /// + /// # Safety + /// + /// `raw` must be this owner's live registered shrinker, and `sc` must + /// point to the initialized control supplied by the shrinker core. unsafe extern "C" fn scan( raw: *mut kernel::bindings::shrinker, sc: *mut kernel::bindings::shrink_control, ) -> kernel::ffi::c_ulong { // SAFETY: Both pointers are supplied by the registered shrinker core. let owner = unsafe { &*((*raw).private_data as *const Self) }; - let target = unsafe { (*sc).nr_to_scan } as usize; - owner.vm.reclaim_idle_pages(target) as _ + // SAFETY: The shrinker core supplies a live initialized control. + let target = unsafe { (*sc).nr_to_scan }; + owner.vm.reclaim_idle_pages(target) } } @@ -348,7 +363,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) { return 0; } @@ -361,11 +376,17 @@ impl Vm { // this reservation; linked aliases retain their borrowed BO and GEM. let _ = unsafe { inner.for_each_private_bo(|_, bo| { let object = bo.obj(); - // Purgeable objects count on busy VMs too; idle backing only on idle ones. - if object.idle_reclaim_candidate() - && (object.madv_locked() > 0 || idle) - && object.sole_sg_lease_releasable_locked() - { + // Count retained backing on idle VMs, and purgeable contents even if idle + // reclaim already released the table. Never wait on the BO mutex under resv. + let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; + let advised = object.madv_locked() > 0; + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + (advised || object.has_sg_table_locked()) + && object.unleased_pages_releasable_locked() + }; + if object.idle_reclaim_candidate() && (advised || idle) && releasable { pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(true) @@ -385,7 +406,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) { return 0; } @@ -396,7 +417,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return 0; }; - let contexts = shared.contexts_naming_root(inner.ttb()); + let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); let mut purged = 0usize; // SAFETY: The private reservation excludes binding edits and restoration. // Callbacks change PTEs/flags only; all aliases stay linked. @@ -408,9 +429,12 @@ impl Vm { } { let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - if segments.is_some() && !object.sole_sg_lease_releasable_locked() { - return Ok(true); - } + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + object.unleased_pages_releasable_locked() + }; + if !releasable { return Ok(true); } } // Any mutation that does not end in a completed purge leaves // INVALIDATED aliases with cleared PTEs while madv stays >= 0, so @@ -467,7 +491,7 @@ impl Vm { if self .status .as_ref() - .map_or(true, |status| status.get() != 0) + .is_none_or(|status| status.get() != 0) || !self .lifetime .as_ref() @@ -482,7 +506,7 @@ impl Vm { let Some(shared) = uat.shared.try_lock() else { return purged; }; - let contexts = shared.contexts_naming_root(inner.ttb()); + let contexts = shared.contexts_naming_root(inner.ttb(), inner.uat_inner.m4_adopted); let mut unpinned = 0usize; // SAFETY: Exclusive residency and the private reservation exclude jobs, // binding edits and retirement. Callbacks change PTEs/flags only; all @@ -494,9 +518,14 @@ impl Vm { { // Avoid inversion against preparation (VmBoInner -> resv). let Some(segments) = bo.data().segments.try_lock() else { return Ok(true); }; - if segments.is_none() || !object.sole_sg_lease_releasable_locked() { - return Ok(true); - } + let releasable = if segments.is_some() { + object.sole_sg_lease_releasable_locked() + } else { + // A competing lease may have made the previous release fail after we + // dropped our segments. Retry that table, but do not count empty backing. + object.has_sg_table_locked() && object.unleased_pages_releasable_locked() + }; + if !releasable { return Ok(true); } } // The association retains SG backing until *all* aliases have // been invalidated and their device translations flushed. A diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index b9e2975c898ff..fec7a6b61ad8f 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -63,7 +63,7 @@ impl Vm { Binding { start: address, end, - object: gem as *const gem::Object as usize, + object: core::ptr::from_ref(gem) as usize, offset, bind_sequence, }, @@ -99,7 +99,7 @@ impl Vm { if shared.count.load(Ordering::Relaxed) == 0 { return; } - let object = gem as *const gem::Object as usize; + let object = core::ptr::from_ref(gem) as usize; let mut bindings = shared.bindings.lock(); bindings.retain(|entry| entry.object != object || entry.bind_sequence > sequence); shared.count.store(bindings.len() as u32, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index 38a2268137eff..a66a82d92a583 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -409,7 +409,7 @@ impl> GpuObject { }; let q = &*ret.inner as *const T; // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; + unsafe { crate::util::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; Ok(ret) } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index b88353a2f7344..c5a47c24ec928 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -18,7 +18,8 @@ use kernel::{ addr::PhysicalAddr, error::Result, page::Page, - prelude::*, // + prelude::*, + sync::Arc, // }; use crate::debug::*; @@ -53,19 +54,20 @@ const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; -/// Size of the smallest CPU data cache line (`CTR_EL0.DminLine`). +/// The system-wide minimum CPU line size, including heterogeneous CPUs. A local +/// CTR_EL0 read can select too large a stride if the task migrates during the loop. #[inline(always)] fn dcache_line_size() -> usize { - let ctr: u64; - // SAFETY: CTR_EL0 is readable at EL1 and reading it has no side effects. - unsafe { - core::arch::asm!( - "mrs {ctr}, ctr_el0", - ctr = out(reg) ctr, - options(nomem, nostack, preserves_flags) - ) - }; - 4 << ((ctr >> 16) & 0xf) + use kernel::bindings; + let ctr = (bindings::SYS_CTR_EL0_Op0 << 19) + | (bindings::SYS_CTR_EL0_Op1 << 16) + | (bindings::SYS_CTR_EL0_CRn << 12) + | (bindings::SYS_CTR_EL0_CRm << 8) + | (bindings::SYS_CTR_EL0_Op2 << 5); + // SAFETY: Read-only lookup of the arm64 sanitized feature register. DminLine + // uses the architecture's LOWER_SAFE merge policy across all admitted CPUs. + let value = unsafe { bindings::read_sanitised_ftr_reg(ctr) }; + 4 << ((value >> bindings::CTR_EL0_DminLine_SHIFT) & 0xf) } /// Cleans the CPU data cache lines covering `len` bytes at `ptr` to the point of coherency, for an @@ -95,7 +97,6 @@ pub(crate) fn clean_poc(ptr: *const u8, len: usize) { /// Cleans and invalidates the CPU data cache lines covering `len` bytes at `ptr` to the point of /// coherency, so that later CPU reads observe what a non-snooping agent wrote. Lines the CPU wrote /// are written back first. [`poc_barrier`] completes the maintenance. -#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { if len == 0 { return; @@ -116,20 +117,114 @@ pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { } } -/// Completes earlier point-of-coherency maintenance for the outer shareable domain (`dsb osh`). +/// Completes earlier point-of-coherency maintenance for the whole system (`dsb sy`). pub(crate) fn poc_barrier() { // SAFETY: Barriers are always safe. - unsafe { core::arch::asm!("dsb osh", options(nostack, preserves_flags)) }; + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; +} + +/// Page tables in bootloader-reserved memory that the kernel does not map ("no-map"), reached +/// through a write-back mapping instead of `struct page` (the `pagetables` and `l2-pagetables` +/// regions, where the firmware builds its root and kernel-window tables). +pub(crate) struct ReservedTables { + regions: KVec, + starts: KVec, +} + +// SAFETY: The mappings are plain write-back memory owned for the lifetime of this object; table +// entries are only accessed as atomics. +unsafe impl Send for ReservedTables {} +// SAFETY: As above. +unsafe impl Sync for ReservedTables {} + +impl ReservedTables { + pub(crate) fn new() -> Self { + Self { + regions: KVec::new(), + starts: KVec::new(), + } + } + + /// Adds a reserved region, mapped write-back. + /// + /// # Safety + /// + /// `res` must be a bootloader-reserved region outside System RAM that stays reserved. + pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { + let start = res.start(); + // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. + let map = unsafe { + kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? + }; + self.starts.push(start, GFP_KERNEL)?; + self.regions.push(map, GFP_KERNEL)?; + Ok(()) + } + + /// The CPU address of a page-table page in one of the regions. + fn lookup(&self, phys: PhysicalAddr) -> Option<*mut u8> { + for (start, map) in self.starts.iter().zip(self.regions.iter()) { + let Some(offset) = phys.checked_sub(*start) else { + continue; + }; + let offset = offset as usize; + if offset + UAT_PGSZ <= map.size() { + // SAFETY: The offset and one page lie inside the mapping. + return Some(unsafe { map.ptr().add(offset) }); + } + } + None + } + + /// Whether `phys` is a page of one of the regions. + pub(crate) fn contains(&self, phys: PhysicalAddr) -> bool { + self.lookup(phys).is_some() + } + + /// Drops the CPU's cached lines of every region, so that later reads see what the firmware + /// wrote while booting. Lines the host dirtied are written back first, never discarded. + pub(crate) fn invalidate(&self) { + for map in self.regions.iter() { + clean_invalidate_poc(map.ptr(), map.size()); + } + poc_barrier(); + } +} + +/// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the +/// reserved mapping when the page lies in a reserved region, else through `struct page`. +/// +/// # Safety +/// +/// `phys` must be a page-table page that stays allocated while `f` runs. +unsafe fn with_table( + reserved: Option<&ReservedTables>, + phys: PhysicalAddr, + offset: usize, + len: usize, + f: impl FnOnce(*mut u8) -> Result, +) -> Result { + if let Some(base) = reserved.and_then(|r| r.lookup(phys)) { + if offset.checked_add(len).ok_or(EINVAL)? > UAT_PGSZ { + return Err(EINVAL); + } + // SAFETY: lookup() checked that the whole page is mapped. + return f(unsafe { base.add(offset) }); + } + // SAFETY: The caller guarantees an allocated page-table page in RAM. + let page = unsafe { Page::borrow_phys_unchecked(&phys) }; + page.with_pointer_into_page(offset, len, f) } /// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. /// /// # Safety /// -/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` -/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they -/// are only exposed as atomics. +/// `ttb` must be the physical address of a page table root, in RAM or in one of the `reserved` +/// regions, that stays allocated while `f` runs. Other agents (the GPU firmware) may access the +/// entries concurrently, which is why they are only exposed as atomics. pub(crate) unsafe fn with_root_entries( + reserved: Option<&ReservedTables>, ttb: PhysicalAddr, count: usize, f: impl FnOnce(&[AtomicU64]) -> R, @@ -137,15 +232,18 @@ pub(crate) unsafe fn with_root_entries( if count > UAT_NPTE { return Err(EINVAL); } - // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. - let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; - page.with_pointer_into_page(0, count * PTE_SIZE, |p| { - // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page - // table entries are naturally aligned. - Ok(f(unsafe { - core::slice::from_raw_parts(p as *const AtomicU64, count) - })) - }) + // SAFETY: A root outside the reserved regions must have a struct page. + if !reserved.is_some_and(|r| r.contains(ttb)) && unsafe { Page::borrow_phys(&ttb) }.is_none() { + return Err(EIO); + } + // SAFETY: The caller guarantees an allocated root page, in RAM or in a reserved region. + unsafe { + with_table(reserved, ttb, 0, count * PTE_SIZE, |p| { + // SAFETY: with_table() checked that `count` entries are mapped, and page table entries + // are naturally aligned. + Ok(f(core::slice::from_raw_parts(p as *const AtomicU64, count))) + }) + } } const PTE_TYPE_BITS: u64 = 3; @@ -253,6 +351,8 @@ impl Prot { Self { memattr, ..*self } } + /// The protection bits of a leaf with this protection (no address, table type or non-global + /// bits), for checking that a firmware leaf word round-trips through [`Self::from_pte`]. #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) const fn leaf_bits(&self) -> u64 { self.as_pte() @@ -278,6 +378,22 @@ impl Prot { (!need_read || readable) && (!need_write || writable) } + /// The native form of a GPU-only permission: AP 0 with the access in PXN (read) and UXN + /// (write), as [`Self::allows_gpu`] reads AP 0. The logical selectors RO, WO, RW, NA map to + /// the native RO, WO, RW, NA. Firmware permissions are returned unchanged. + const fn native_gpu(self) -> Self { + if self.ap != AP_GPU { + return self; + } + let selector = (self.high_bits >> 1) & 3; + Self { + ap: AP_FW_GPU, + high_bits: (self.high_bits & !(HIGH_BITS_PXN | HIGH_BITS_UXN)) + | (((selector + 1) & 3) << 1), + ..self + } + } + pub(crate) const fn is_cached_noncoherent(&self) -> bool { self.ap != AP_GPU && self.memattr == MEMATTR_CACHED } @@ -296,6 +412,12 @@ impl Default for Prot { pub(crate) struct UatPageTable { ttb: PhysicalAddr, ttb_owned: bool, + /// Ownership could not be relinquished to firmware; physical tables remain until reboot. + retained: bool, + /// Reserved regions holding tables this page table may reach. + reserved: Option>, + /// Store GPU-only permissions in the native AP 0 form ([`Prot::native_gpu`]). + native_gpu: bool, /// Tables below a root the driver did not allocate may belong to the bootloader or the /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the /// tables below the root, as with the handoff firmware interface. @@ -305,7 +427,6 @@ pub(crate) struct UatPageTable { oas_mask: u64, /// Clean new tables and changed entries to the point of coherency, for a UAT walker that does /// not snoop the CPU caches ([`Self::enable_clean_poc`]). - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: bool, } @@ -317,45 +438,49 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: true, + retained: false, + reserved: None, + native_gpu: false, driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } /// Makes every later change of this page table visible to a UAT walker that does not snoop the - /// CPU caches (G15), and cleans the root, which was zeroed through the cacheable linear map. + /// CPU caches (G16, and `UatQuirks::clean_tables`), and cleans the root, which may have been + /// zeroed through the cacheable linear map. /// /// From now on, a new table page is cleaned before its parent descriptor names it, a changed /// table or leaf descriptor is cleaned after the store, and every mutating walk ends with a - /// `dsb osh`. Call it before anything is mapped and before any context names the root. - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + /// `dsb sy`. Call it before anything is mapped and before any context names the root. pub(crate) fn enable_clean_poc(&mut self) -> Result { self.clean_poc = true; - // SAFETY: The root is a RAM page owned or borrowed by this page table for its lifetime. - let root = unsafe { Page::borrow_phys(&self.ttb) }.ok_or(EIO)?; - root.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; + // SAFETY: The root is owned or borrowed by this page table for its lifetime, in RAM or in + // a reserved region. + unsafe { + with_table(self.reserved.as_deref(), self.ttb, 0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + }) + }?; poc_barrier(); Ok(()) } - /// Whether changes are cleaned to the point of coherency (always false without UAT adoption support). + /// Whether changes are cleaned to the point of coherency. #[inline(always)] fn cleans_poc(&self) -> bool { - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] - { - self.clean_poc - } - #[cfg(not(CONFIG_DRM_ASAHI_UAT_ADOPTION))] - { - false - } + self.clean_poc + } + + /// Stores GPU-only permissions of this (client or kernel-lower) table in the native AP 0 form + /// (`UatQuirks::native_gpu_permissions`). Firmware roots never hold GPU-only mappings. + pub(crate) fn with_native_gpu(mut self, native_gpu: bool) -> Self { + self.native_gpu = native_gpu; + self } /// Borrow a root; track individual child ownership only when other agents @@ -366,6 +491,7 @@ impl UatPageTable { ias: u32, oas: u32, shared_tables: bool, + reserved: Option>, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -381,7 +507,9 @@ impl UatPageTable { return Err(EINVAL); } // SAFETY: The bootloader-managed root stays valid while this page table exists. - if unsafe { Page::borrow_phys(&ttb) }.is_none() { + let reachable = reserved.as_ref().is_some_and(|r| r.contains(ttb)) + || unsafe { Page::borrow_phys(&ttb) }.is_some(); + if !reachable { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", ttb @@ -392,15 +520,23 @@ impl UatPageTable { Ok(UatPageTable { ttb, ttb_owned: false, + retained: false, + reserved, + native_gpu: false, driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] clean_poc: false, }) } + /// Retains every physical table after an uncertain M4 handoff. No other agent may reclaim + /// this tree merely because a userspace handle closed or a fence reported an error. + pub(crate) fn retain(&mut self) { + self.retained = true; + } + pub(crate) fn ttb(&self) -> PhysicalAddr { self.ttb } @@ -470,6 +606,7 @@ impl UatPageTable { // Handle the case where iova_range.end is just at the top boundary of the IAS let end = ((iova_range.end - 1) & self.ias_mask) + 1; + let reserved = self.reserved.clone(); let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); // For every child table, retain the parent page and entry that names it. // Children are detached before their parents are freed, so these borrowed @@ -513,55 +650,58 @@ impl UatPageTable { ); let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) as usize; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); - pt_addr[level] = - upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { - let uptep = p as *const _ as *const Pte; - // SAFETY: with_pointer_into_page() ensures the pointer is valid, - // and our index is aligned so it is safe to deref as an AtomicU64. - let upte = unsafe { &*uptep }; - let mut upte_val = upte.load(Ordering::Relaxed); - // Allocate if requested - if upte_val == 0 && alloc { - let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); - if clean { - // A non-snooping walker must never follow a parent to a - // table whose zeroes are still in the CPU caches. - pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; - } - if let Some(tables) = self.driver_tables.as_mut() { - tables.push(pt_page.phys(), GFP_KERNEL)?; - } - let pt_paddr = Page::into_phys(pt_page); - upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; - // Complete child-table initialization before a - // concurrent device walk can follow its parent. - // A later mapping/TLBI barrier cannot order an - // already-visible parent against earlier clears. - crate::mem::sync(); - upte.store(upte_val, Ordering::Relaxed); - if clean { - clean_poc(uptep.cast(), PTE_SIZE); - } + let driver_tables = &mut self.driver_tables; + let oas_mask = self.oas_mask; + let entry = |p: *mut u8| -> Result> { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_table() ensures the pointer is valid, and our index is + // aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if clean { + // A non-snooping walker must never follow a parent to a + // table whose zeroes are still in the CPU caches. + pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; + } + if let Some(tables) = driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; } - if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) - } else if upte_val == 0 || (!alloc && !free) { - mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); - Ok(None) - } else { - pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, - iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); - Ok(None) + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); + upte.store(upte_val, Ordering::Relaxed); + if clean { + clean_poc(uptep.cast(), PTE_SIZE); } - })?; + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + pt_addr[level] = unsafe { + with_table(reserved.as_deref(), phys, upidx * PTE_SIZE, PTE_SIZE, entry) + }?; if pt_addr[level].is_some() { pt_parent[level] = Some((phys, upidx)); } @@ -597,13 +737,10 @@ impl UatPageTable { count, iova ); - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; - pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let leaves = |p: *mut u8| -> Result { let ptep = p as *const _ as *const Pte; // SAFETY: We know this is a valid pointer to PTEs and the range is valid and - // checked by with_pointer_into_page(). + // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; let result = cb(iova, ptes); if clean && mutate { @@ -611,7 +748,12 @@ impl UatPageTable { clean_poc(p, count * PTE_SIZE); } result - })?; + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + unsafe { + with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); iova = align(iova + 1, block); @@ -636,16 +778,9 @@ impl UatPageTable { } /// Free a page table, unless it was not allocated by the driver. - fn free_table( - &mut self, - phys: PhysicalAddr, - parent: Option<(PhysicalAddr, usize)>, - ) -> Result { + fn free_table(&mut self, phys: PhysicalAddr, parent: Option<(PhysicalAddr, usize)>) -> Result { let owned = if let Some(tables) = self.driver_tables.as_ref() { - match tables.iter().position(|&table| table == phys) { - Some(_) => true, - None => false, - } + tables.contains(&phys) } else { true }; @@ -653,32 +788,44 @@ impl UatPageTable { return Ok(()); } if let Some((parent_phys, parent_index)) = parent { - // SAFETY: The bottom-up traversal keeps the parent allocated while - // its child descriptor is detached. A changed descriptor indicates - // an ownership violation; fail closed instead of freeing a table - // that may still be reachable. - let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; - page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { + let oas_mask = self.oas_mask; + let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + // SAFETY: Only looks the page up; nothing is accessed through it. + if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { + return Err(EIO); + } + let reserved = self.reserved.as_deref(); + let cleans = self.cleans_poc(); + let detach = |p: *mut u8| -> Result { + // SAFETY: The live page-table page contains aligned u64 PTEs, and with_table() + // bounds this access to one PTE. let pte = unsafe { &*(p as *const AtomicU64) }; let current = pte.load(Ordering::Acquire); - let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); - if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { + let expected = phys & oas_mask & (!UAT_PGMSK as u64); + if current & oas_mask & (!UAT_PGMSK as u64) != expected { return Err(EIO); } pte.store(0, Ordering::Release); - if self.cleans_poc() { + if cleans { clean_poc(p, PTE_SIZE); } Ok(()) - })?; + }; + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; } if let Some(tables) = self.driver_tables.as_mut() { - let index = tables - .iter() - .position(|&table| table == phys) - .ok_or(EIO)?; + let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; tables.swap_remove(index); } + if self.cleans_poc() { + // Complete the parent descriptor clear before its detached child can be reused. + // Callers have already removed this tree from live contexts before freeing tables. + poc_barrier(); + } // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without // `driver_tables`, every child table belongs to the driver. unsafe { Page::from_phys(phys) }; @@ -769,6 +916,15 @@ impl UatPageTable { } } + /// The permission bits as this table stores them. + fn stored(&self, prot: Prot) -> Prot { + if self.native_gpu { + prot.native_gpu() + } else { + prot + } + } + pub(crate) fn map_pages( &mut self, iova_range: Range, @@ -788,6 +944,7 @@ impl UatPageTable { } let pte_bits = self.pte_bits(); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { @@ -814,6 +971,7 @@ impl UatPageTable { iova_range, prot ); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); @@ -861,6 +1019,9 @@ impl UatPageTable { impl Drop for UatPageTable { fn drop(&mut self) { + if self.retained { + return; + } mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); if self .with_pages(self.va_range.clone(), false, true, |iova, ptes| { diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs index a68352828cfbc..20396f2e77225 100644 --- a/drivers/gpu/drm/asahi/queue/common.rs +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -33,7 +33,7 @@ pub(super) fn get_timestamp_object( return Err(ERANGE); } Ok(Some(UserTimestamp { - mapping: mapping.clone(), + mapping: mapping.firmware_mapping()?, offset: offset as usize, })) } else { diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 0872a50ce5d46..4abdd3e485f72 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -262,7 +262,8 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { type TimeoutData = QueueTimeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + const MODULE: Option<&'static kernel::ThisModule> = + Some(&crate::THIS_MODULE); fn timeout_data(&self) -> QueueTimeout { QueueTimeout { dev: self.dev.clone(), id: self.id } @@ -754,7 +755,22 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; - cmdbuf.skip(header.size as usize); + let payload_size = header.size as usize; + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; + if render.ts_frag_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + return Err(EINVAL); + } + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; + if compute.ts_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + return Err(EINVAL); + } + } + _ => cmdbuf.skip(payload_size), + } nr_commands += 1; match header.cmd_type as u32 { diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index fb22569ea5fdf..0a4ee8e7563ec 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -13,10 +13,11 @@ use crate::hw; use core::mem::MaybeUninit; use kernel::{ - bindings, c_str, + bindings, + c_str, device::Core, - devres::Devres, error::{from_err_ptr, to_result}, + devres::Devres, io::{ mem::IoMem, Io, @@ -222,7 +223,10 @@ impl CpuControl { }; // SAFETY: pdev is live; the provider establishes the consumer device link. let mbox = unsafe { - from_err_ptr(bindings::apple_mbox_get(pdev.as_ref().as_raw(), index.try_into()?))? + from_err_ptr(bindings::apple_mbox_get( + pdev.as_ref().as_raw(), + index.try_into()?, + ))? }; let mut lifecycle = MaybeUninit::uninit(); // SAFETY: mbox is live, and lifecycle is writable storage for the C result. @@ -235,7 +239,10 @@ impl CpuControl { } // SAFETY: This only checks the provider's matching start/stop capability. to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(mbox) })?; - Ok(Self { dev: pdev.into(), mbox }) + Ok(Self { + dev: pdev.into(), + mbox, + }) } /// Start through the owner of the CPU registers and runtime-PM reference. @@ -251,7 +258,11 @@ impl CpuControl { let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }); if let Err(error) = result { if error != ENODEV { - dev_err!(self.dev.as_ref(), "Could not stop GPU coprocessor: {:?}\n", error); + dev_err!( + self.dev.as_ref(), + "Could not stop GPU coprocessor: {:?}\n", + error + ); } } } @@ -560,7 +571,10 @@ impl G17Resources { /// Map the required resources given our platform device. pub(crate) fn new(pdev: &platform::Device>) -> Result { let sgx_req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; - let sgx = Arc::new(sgx_req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?; + let sgx = Arc::new( + sgx_req.iomap_sized::()?.into_devres()?, + GFP_KERNEL, + )?; Ok(G17Resources { dev: pdev.into(), @@ -701,7 +715,6 @@ impl G17Resources { let val = sgx.try_read32(G17_GATING_CONTROL)?; sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) } - } /// Split the packed per-cluster core masks into one mask per cluster. @@ -732,11 +745,7 @@ fn split_core_masks( } if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!( - dev.as_ref(), - "Leftover core mask: {:#x?}\n", - core_mask_regs - ); + dev_err!(dev.as_ref(), "Leftover core mask: {:#x?}\n", core_mask_regs); return Err(EIO); } @@ -744,7 +753,10 @@ fn split_core_masks( } /// Decode the silicon revision from the ID version register. -fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { +fn gpu_revision( + dev: &platform::Device, + id_version: u32, +) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { Ok(match (id_version >> 8) & 0xff { 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), @@ -758,3 +770,29 @@ fn gpu_revision(dev: &platform::Device, id_version: u32) -> Result<(hw::GpuRevis } }) } + +/// Device-managed SGX mapping retained by the G16 firmware session. +#[cfg(CONFIG_DRM_ASAHI_G16)] +#[derive(Clone)] +pub(crate) struct G16Resources { + sgx: Arc>>, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl G16Resources { + pub(crate) fn new(pdev: &platform::Device>) -> Result { + let req = pdev.io_request_by_name(c"sgx").ok_or(EINVAL)?; + Ok(Self { + sgx: Arc::new(req.iomap_sized::()?.into_devres()?, GFP_KERNEL)?, + }) + } + pub(crate) fn sgx_read32_at(&self, offset: usize) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + (&*sgx).as_view().try_read32(offset) + } + #[allow(dead_code)] + pub(crate) fn sgx_write32_at(&self, offset: usize, value: u32) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + (&*sgx).as_view().try_write32(value, offset) + } +} diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 9f0baa4350d93..7d7d785a83133 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -32,9 +32,21 @@ struct asahi_stats_snapshot { u64 jobs; }; +/* Mirror of the repr(C) Rust `memacct::Ledger`, in bytes. */ +struct asahi_gpu_memory { + u64 user_committed; + u64 user_resident; + u64 kernel_resident; + u64 refused; + u64 limit; + u64 client_limit; +}; + struct asahi_stats_attribute { struct device_attribute attr; + struct device_attribute memory_attr; const struct asahi_stats_snapshot *snapshot; + const struct asahi_gpu_memory *memory; bool export_enabled; }; @@ -72,20 +84,52 @@ static ssize_t agx_stats_show(struct device *dev, return n; } +/* + * GPU memory held by the driver, in bytes: user objects created (committed) + * and held by the GPU (resident), held kernel objects, the limits that admit + * user objects (0: none) and the number of refused creations. + */ +static ssize_t gpu_memory_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct asahi_stats_attribute *stats = + container_of(attr, struct asahi_stats_attribute, memory_attr); + const struct asahi_gpu_memory *mem = stats->memory; + ssize_t n = 0; + + n += scnprintf(buf + n, PAGE_SIZE - n, "user_committed %llu\n", + (unsigned long long)READ_ONCE(mem->user_committed)); + n += scnprintf(buf + n, PAGE_SIZE - n, "user_resident %llu\n", + (unsigned long long)READ_ONCE(mem->user_resident)); + n += scnprintf(buf + n, PAGE_SIZE - n, "kernel_resident %llu\n", + (unsigned long long)READ_ONCE(mem->kernel_resident)); + n += scnprintf(buf + n, PAGE_SIZE - n, "limit %llu\n", + (unsigned long long)READ_ONCE(mem->limit)); + n += scnprintf(buf + n, PAGE_SIZE - n, "client_limit %llu\n", + (unsigned long long)READ_ONCE(mem->client_limit)); + n += scnprintf(buf + n, PAGE_SIZE - n, "refused %llu\n", + (unsigned long long)READ_ONCE(mem->refused)); + + return n; +} /* * Called from Rust's `AsahiDriver::probe` after the DRM device is * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, void **handle) + int export_enabled, const void *memory, void **handle) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || (export_enabled && !snapshot)) + if (!dev || !handle || !memory || (export_enabled && !snapshot)) return -ENODEV; + BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); + BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); + BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); + BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 40); BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); @@ -102,17 +146,70 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->snapshot = snapshot; stats->export_enabled = export_enabled; + sysfs_attr_init(&stats->memory_attr.attr); + stats->memory_attr.attr.name = "gpu_memory"; + stats->memory_attr.attr.mode = 0444; + stats->memory_attr.show = gpu_memory_show; + stats->memory = memory; + ret = device_create_file(dev, &stats->attr); if (ret) { kfree(stats); return ret; } + ret = device_create_file(dev, &stats->memory_attr); + if (ret) { + device_remove_file(dev, &stats->attr); + kfree(stats); + return ret; + } + *handle = stats; return 0; } EXPORT_SYMBOL_GPL(asahi_sysfs_register); +/* + * G17: GPU idle power-off delay (ms, 2-1000). On T6040/25G83 the firmware powers the GPU cores + * off after 2 ms idle and reads its own delay only at boot; above 2 the driver holds the cores + * powered until the GPU has been idle this long (power::IDLE_HOLD_MS). Each wake-up there costs + * about 1 ms, so userspace may lengthen it on AC power. + */ +static u32 *asahi_idle_word; + +static ssize_t idle_off_delay_ms_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%u\n", READ_ONCE(*asahi_idle_word)); +} + +static ssize_t idle_off_delay_ms_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + u32 ms; + + if (kstrtou32(buf, 0, &ms) || ms < 2 || ms > 1000) + return -EINVAL; + WRITE_ONCE(*asahi_idle_word, ms); + return count; +} + +static DEVICE_ATTR_RW(idle_off_delay_ms); + +int asahi_sysfs_register_idle(struct device *dev, u32 *word) +{ + int ret; + + if (!dev || !word) + return -ENODEV; + asahi_idle_word = word; + ret = device_create_file(dev, &dev_attr_idle_off_delay_ms); + if (ret) + asahi_idle_word = NULL; + return ret; +} +EXPORT_SYMBOL_GPL(asahi_sysfs_register_idle); + void asahi_sysfs_unregister(struct device *dev, void *handle) { struct asahi_stats_attribute *stats = handle; @@ -121,7 +218,12 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) return; /* Removal drains active show callbacks before either allocation dies. */ + device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); + if (asahi_idle_word) { + device_remove_file(dev, &dev_attr_idle_off_delay_ms); + asahi_idle_word = NULL; + } kfree(stats); } EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index 5969a1718925f..f1873f9be6caa 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -6,7 +6,8 @@ struct device; int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, void **handle); + int export_enabled, const void *memory, void **handle); +int asahi_sysfs_register_idle(struct device *dev, u32 *word); void asahi_sysfs_unregister(struct device *dev, void *handle); #endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index 079623d1ef210..cee5d52a4e261 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,7 +3,7 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::stats::StatsSnapshot; +use crate::{memacct::Ledger, stats::StatsSnapshot}; use core::ffi::{c_int, c_void}; extern "C" { @@ -11,12 +11,30 @@ extern "C" { dev: *mut c_void, snapshot: *const StatsSnapshot, export_enabled: c_int, + memory: *const Ledger, handle: *mut *mut c_void, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } +extern "C" { + fn asahi_sysfs_register_idle(dev: *mut c_void, word: *mut u32) -> c_int; +} + +/// Export the idle power-off delay as the read-write `idle_off_delay_ms` (G17). +pub(crate) fn register_idle( + dev: *mut kernel::bindings::device, + word: *mut u32, +) -> kernel::error::Result { + // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes + // it with READ_ONCE/WRITE_ONCE. + let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; + if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { Ok(()) } +} + /// Register only after initialization succeeds, keeping the snapshot Arc alive. +/// +/// Also registers `gpu_memory`, which shows the static GPU memory ledger. pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, @@ -30,6 +48,7 @@ pub(crate) fn register( dev.cast(), snapshot, c_int::from(export_enabled), + &crate::memacct::LEDGER, &mut handle, ) }; diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs new file mode 100644 index 0000000000000..976ec9f6bf48b --- /dev/null +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Prepared timestamp destinations shared by native command completion paths. + +use crate::mmu; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use crate::{file, gem}; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use core::ops::Range; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use core::sync::atomic::{fence as barrier, Ordering}; +use kernel::{prelude::*, sync::Arc}; +#[cfg(CONFIG_DRM_ASAHI_G16)] +use kernel::{uapi, xarray}; + +/// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. +/// CPU-retired timestamps retain their backing without consuming global firmware address space. +pub(crate) enum Buffer { + Firmware(Arc), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Cpu { + // The GEM pin keeps the object's held SG table alive through CPU publication. + gem: gem::KernelMappingPin, + range: Range, + }, +} + +impl Buffer { + pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { + Ok(Self::Firmware(Arc::new(mapping, GFP_KERNEL)?)) + } + + /// Retain a CPU-written byte range without publishing any firmware address. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn cpu(bo: gem::ObjectRef, range: Range) -> Result { + let size = range.end.checked_sub(range.start).ok_or(EINVAL)?; + if size == 0 || (range.start | size) & mmu::UAT_PGMSK != 0 || range.end > bo.size() { + return Err(EINVAL); + } + let gem = gem::KernelMappingPin::new(&bo.gem); + let registered = bo.gem.dev().registration_guard().ok_or(ENODEV)?; + let parent: &kernel::platform::Device = registered.as_ref(); + let _ = gem::held_sg_table(&gem, parent.as_ref())?; + Ok(Self::Cpu { gem, range }) + } + + pub(crate) fn size(&self) -> usize { + match self { + Self::Firmware(mapping) => mapping.size(), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { range, .. } => range.end - range.start, + } + } + + pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + match self { + Self::Firmware(mapping) => mapping.timestamp_mapping(offset), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { gem, range, .. } => { + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { + return Err(ERANGE); + } + mmu::TimestampMapping::new(gem, range.start.checked_add(offset).ok_or(EOVERFLOW)?) + } + } + } + + /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. + pub(crate) fn firmware_mapping(&self) -> Result> { + match self { + Self::Firmware(mapping) => Ok(mapping.clone()), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => Err(EINVAL), + } + } +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +fn checked_offset(offset: u64, size: usize, aligned: bool) -> Result { + let offset = usize::try_from(offset).map_err(|_| EINVAL)?; + let end = offset.checked_add(size_of::()).ok_or(EINVAL)?; + if aligned && offset % size_of::() != 0 { + return Err(EINVAL); + } + if end > size { + return Err(if aligned { EINVAL } else { ERANGE }); + } + Ok(offset) +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +struct Destination { + // Retain the bound range and backing along with its prepared CPU view. + _mapping: Arc, + cpu: mmu::TimestampMapping, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl Destination { + fn resolve( + objects: Pin<&xarray::XArray>>, + timestamp: uapi::drm_asahi_timestamp, + aligned: bool, + ) -> Result> { + if timestamp.handle == 0 { + return Ok(None); + } + let object = objects + .lock() + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + let file::Object::TimestampBuffer(mapping) = object; + let offset = checked_offset(timestamp.offset.into(), mapping.size(), aligned)?; + let cpu = mapping.timestamp_mapping(offset)?; + Ok(Some(Self { + _mapping: mapping, + cpu, + })) + } + + fn write(&self, ticks: u64, frequency_hz: u64) { + const NSEC_PER_SEC: u64 = 1_000_000_000; + let ns = (ticks / frequency_hz) + .wrapping_mul(NSEC_PER_SEC) + .wrapping_add(ticks % frequency_hz * NSEC_PER_SEC / frequency_hz); + self.cpu.write(ns); + barrier(Ordering::Release); + } +} + +/// Timestamp mappings resolved and retained before any command is enqueued. +#[cfg(CONFIG_DRM_ASAHI_G16)] +pub(crate) struct Destinations(Option; 4]>>); + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl Destinations { + /// G16 retains its eight-byte destination alignment and EINVAL contract. + pub(crate) fn resolve_aligned( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + ) -> Result { + Self::resolve_inner(objects, timestamps, true) + } + + fn resolve_inner( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + aligned: bool, + ) -> Result { + if timestamps.is_empty() || timestamps.len() > 2 { + return Err(EINVAL); + } + let mut destinations = core::array::from_fn(|_| None); + for (index, timestamp) in timestamps.iter().enumerate() { + destinations[2 * index] = Destination::resolve(objects, timestamp.start, aligned)?; + destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end, aligned)?; + } + // Empty submissions need no cleanup allocation. Timestamped commands + // prepare the unique work item before any output fence is published. + let retained = if destinations.iter().any(Option::is_some) { + Some(crate::cleanup::Deferred::new(destinations)?) + } else { + None + }; + Ok(Self(retained)) + } + + /// Publish measured timestamps before signalling completion or releasing dependent work. + pub(crate) fn publish(&self, ticks: [u64; 4], frequency_hz: u64) { + for (destination, ticks) in self.iter().zip(ticks) { + if let Some(destination) = destination { + destination.write(ticks, frequency_hz); + } + } + } + + fn iter(&self) -> core::slice::Iter<'_, Option> { + let destinations: &[Option] = self + .0 + .as_deref() + .map_or(&[], |destinations| destinations.as_slice()); + destinations.iter() + } +} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index b51b027eb573c..a5f9455b23f5f 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -217,3 +217,17 @@ mod tests { assert!(reader.read_supported::(8, 8).is_err()); } } + +/// Fallibly initializes `slot` with `init` (`pin_init::raw_try_init` of later pin-init releases). +/// +/// # Safety +/// +/// - `slot` is a valid pointer to uninitialized memory. +/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to +/// deallocate. +/// - `slot` will not move until it is dropped, i.e. it will be pinned. +#[inline(always)] +pub(crate) unsafe fn raw_try_init(slot: *mut T, init: impl PinInit) -> Result<(), E> { + // SAFETY: Per safety requirement. + unsafe { init.__pinned_init(slot) } +} diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index c00da634972e5..9de2e7f0f4f97 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -87,8 +87,16 @@ static int __drm_gem_shmem_init(struct drm_device *dev, struct drm_gem_shmem_obj * why this is required _and_ expected if you're * going to pin these pages. */ - mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER | - __GFP_RETRY_MAYFAIL | __GFP_NOWARN); + gfp_t gfp = GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN; + + /* A bulk pin must not run direct reclaim once per folio when + * the driver can instead return an allocation failure. Clearing + * DIRECT_RECLAIM also makes memcg charges fail without reclaim + * or invoking the OOM killer. Preserve placement and kswapd. + */ + if (shmem->pages_no_reclaim) + gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_RETRY_MAYFAIL); + mapping_set_gfp_mask(obj->filp->f_mapping, gfp); } return 0; @@ -612,10 +620,10 @@ bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shm struct drm_gem_object *obj = &shmem->base; dma_resv_assert_held(obj->resv); - return shmem->sgt && !drm_gem_is_imported(obj) && !obj->dma_buf && + return !drm_gem_is_imported(obj) && !obj->dma_buf && shmem->madv >= 0 && !refcount_read(&shmem->pages_pin_count) && !refcount_read(&shmem->vmap_use_count) && - refcount_read(&shmem->pages_use_count) == 1 && + refcount_read(&shmem->pages_use_count) == !!shmem->sgt && dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP); } EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); @@ -1075,6 +1083,10 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ if (shmem->sgt) return shmem->sgt; + /* Purged contents cannot be restored by rebuilding an empty table. */ + if (shmem->madv < 0) + return ERR_PTR(-EINVAL); + drm_WARN_ON(obj->dev, drm_gem_is_imported(obj)); ret = drm_gem_shmem_get_pages_locked(shmem); diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index d25bca5352aa6..30791d328550d 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -37,6 +37,13 @@ struct drm_gem_shmem_object { /** @reclaimable_cpu_mappings: CPU VMAs may be revoked by the driver. */ bool reclaimable_cpu_mappings; + /** + * @pages_no_reclaim: Disable direct reclaim during backing population. + * Set before drm_gem_shmem_init(). Drivers that can fail a large pin + * request cheaply may opt in; background reclaim remains enabled. + */ + bool pages_no_reclaim; + /** * @cpu_pages: Pages populated one at a time by CPU faults on revocable * VMAs while no device mapping holds @pages. Sparse; under base.resv. @@ -145,7 +152,9 @@ int drm_gem_shmem_madvise_locked(struct drm_gem_shmem_object *shmem, int madv); static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem) { return (shmem->madv > 0) && - !refcount_read(&shmem->pages_pin_count) && shmem->sgt && + !refcount_read(&shmem->pages_pin_count) && + !refcount_read(&shmem->vmap_use_count) && + refcount_read(&shmem->pages_use_count) == !!shmem->sgt && !shmem->base.dma_buf && !drm_gem_is_imported(&shmem->base); } diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index d311b92620b86..66396f89cc333 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -212,6 +212,20 @@ struct drm_asahi_params_global { * DRM_ASAHI_GPU_HAL_200. */ __u32 gpu_hal_generation; + + /** + * @memory_limit: Device-wide limit on user committed plus kernel + * resident bytes. Zero means no driver limit. This is a capacity, + * not current free memory; other clients share it. + */ + __u64 memory_limit; + + /** + * @client_memory_limit: Per-DRM-file limit on user committed bytes. + * Zero means no driver limit. Userspace must zero-initialize the reply + * buffer: older kernels leave these optional tail fields untouched. + */ + __u64 client_memory_limit; }; /** @@ -348,6 +362,12 @@ enum drm_asahi_feature { * accept render_scs_layout. */ DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 15, + + /** + * @DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD: Compute commands accept + * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. + */ + DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -806,7 +826,8 @@ enum drm_asahi_priority { /** * @DRM_ASAHI_PRIORITY_HIGH: High priority queue. * - * Reserved for future extension. + * Requires CAP_SYS_NICE or DRM master; otherwise queue creation fails + * with -EACCES. */ DRM_ASAHI_PRIORITY_HIGH = 2, @@ -1614,6 +1635,17 @@ enum drm_asahi_compute_flags { DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF = (1L << 0), DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), + + /** + * @DRM_ASAHI_COMPUTE_AS_BUILD: Acceleration-structure build work, + * including compute-based transfers. Requires BOUNDED_AS_BUILD. + * This is an execution hint, not proof of progress or permission to + * access additional memory. The kernel validates the command normally + * and caps its execution allowance at ten seconds (four extra 2s polls). + * Other commands in the same submission receive the same allowance so + * ordered helpers cannot time out while waiting for the build. + */ + DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2), }; /** diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 099e64b3ce5fd..9ab7494251e28 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -30,6 +30,7 @@ #include #include +#include #include #include #include diff --git a/rust/helpers/mm.c b/rust/helpers/mm.c index b5540997bd20d..d9933c4595682 100644 --- a/rust/helpers/mm.c +++ b/rust/helpers/mm.c @@ -23,6 +23,18 @@ __rust_helper bool rust_helper_mmget_not_zero(struct mm_struct *mm) return mmget_not_zero(mm); } +__rust_helper void rust_helper_add_mm_counter(struct mm_struct *mm, int member, + long value) +{ + add_mm_counter(mm, member, value); +} + +__rust_helper unsigned long rust_helper_get_mm_counter_sum(struct mm_struct *mm, + int member) +{ + return get_mm_counter_sum(mm, member); +} + __rust_helper void rust_helper_mmap_read_lock(struct mm_struct *mm) { mmap_read_lock(mm); diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 4d6674f245049..e63b62b54a17d 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -192,7 +192,7 @@ impl UnregisteredDevice { dumb_create: T::Object::ALLOC_OPS.dumb_create, dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, - show_fdinfo: None, + show_fdinfo: Some(drm::File::::show_fdinfo_callback), fbdev_probe: None, major: T::INFO.major, diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 8a56f00203f56..88709a7f9b36d 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -12,6 +12,23 @@ use crate::{ }; use core::marker::PhantomData; +/// Memory a client reports in its fdinfo, in bytes, as `drm--system` lines. +/// +/// See `Documentation/gpu/drm-usage-stats.rst`. `None` leaves an optional key out. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct MemoryStats { + /// Memory shared with other clients (`drm-shared-*`, also part of `drm-total-*`). + pub shared: u64, + /// Memory private to the client (part of `drm-total-*`). + pub private: u64, + /// Memory currently resident (`drm-resident-*`). + pub resident: Option, + /// Resident memory userspace marked purgeable (`drm-purgeable-*`). + pub purgeable: Option, + /// Memory in use by active submissions (`drm-active-*`). + pub active: Option, +} + /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { /// The parent `Driver` implementation for this `DriverFile`. @@ -22,6 +39,11 @@ pub trait DriverFile { /// Get raw drm_file pointer fn as_raw(&self) -> *mut bindings::drm_file; + + /// Memory statistics for this client's fdinfo. The default, `None`, prints no memory keys. + fn memory_stats(&self) -> Option { + None + } } /// An open DRM File. @@ -48,6 +70,12 @@ impl File { self.0.get() } + /// Whether this file holds the device's DRM master (the compositor or display server). + pub fn is_current_master(&self) -> bool { + // SAFETY: By the type invariants of `Self`, `self.as_raw()` is a valid open `drm_file`. + unsafe { bindings::drm_is_current_master(self.as_raw()) } + } + /// Tag recent feeder work for temporary capacity hints during syncobj waits. /// User scheduling attributes and CPU affinity remain unchanged. pub fn set_syncobj_wait_hint(&self, util_min: u32, duration_ms: u32) { @@ -99,6 +127,47 @@ impl File { 0 } + /// The `show_fdinfo` callback of a `struct drm_driver`. + pub(crate) extern "C" fn show_fdinfo_callback( + printer: *mut bindings::drm_printer, + raw_file: *mut bindings::drm_file, + ) { + // SAFETY: `drm_show_fdinfo()` passes an open `struct drm_file` of this driver, whose + // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the + // file, so `postclose_callback()` cannot run concurrently. + let file = unsafe { File::::from_raw(raw_file) }; + let Some(stats) = file.inner().memory_stats() else { + return; + }; + + let mut supported = 0; + let mut raw = bindings::drm_memory_stats { + shared: stats.shared, + private: stats.private, + resident: 0, + purgeable: 0, + active: 0, + }; + if let Some(resident) = stats.resident { + raw.resident = resident; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_RESIDENT; + } + if let Some(purgeable) = stats.purgeable { + raw.purgeable = purgeable; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_PURGEABLE; + } + if let Some(active) = stats.active { + raw.active = active; + supported |= bindings::drm_gem_object_status_DRM_GEM_OBJECT_ACTIVE; + } + + // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` + // and the region name outlive the call. + unsafe { + bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) + }; + } + /// The postclose callback of a `struct drm_file`. pub(crate) extern "C" fn postclose_callback( _raw_dev: *mut bindings::drm_device, diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index fb7e5c38e5483..83eca852139be 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -82,6 +82,12 @@ pub struct ObjectConfig<'a, T: DriverObject> { /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. pub reclaimable_cpu_mappings: bool, + /// Fail backing population when it would need direct reclaim. + /// + /// Background reclaim and ordinary allocation placement are preserved. + /// Intended for large, fallible pin requests, not essential kernel backing. + pub pages_no_reclaim: bool, + /// Reuse the DMA reservation from another GEM object. /// /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. @@ -94,13 +100,13 @@ impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> { Self { map_wc: false, reclaimable_cpu_mappings: false, + pages_no_reclaim: false, parent_resv_obj: None, } } } -const SGT_PERMANENT: usize = 1 << (usize::BITS - 1); -const SGT_RECLAIMING: usize = 1 << (usize::BITS - 2); +const SGT_RECLAIMING: usize = 1 << (usize::BITS - 1); const SGT_USERS: usize = SGT_RECLAIMING - 1; /// A shmem-backed GEM object. @@ -116,12 +122,12 @@ pub struct Object { obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, - /// Devres object for unmapping any SGTable on driver-unbind. - sgt_res: ManuallyDrop>>>, + /// One cleanup registration for the C SG-table cache; never holds a table borrow. + sgt_cleanup: ManuallyDrop>>>, #[pin] - /// Lock for protecting initialization of `sgt_res`. + /// Serializes cleanup registration and table acquisition. sgt_lock: Mutex<()>, - /// Owned SGT readers, or permanent borrowed access, exclude backing eviction. + /// Counted SG leases exclude backing eviction; the high bit excludes new leases. sgt_access: AtomicUsize, #[pin] inner: T, @@ -189,13 +195,13 @@ impl Object { // `Object` let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut(); - // We need to drop `sgt_res` first, since doing so requires that the GEM object is still + // We need to drop `sgt_cleanup` first, since doing so requires that the GEM object is still // alive. // SAFETY: // - We verified above that `this` is valid. // - We are in free_callback, guaranteeing we have exclusive access to `this` and that - // `sgt_res` will not be used after dropping it here. - unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) }; + // `sgt_cleanup` will not be used after dropping it here. + unsafe { ManuallyDrop::drop(&mut (*this).sgt_cleanup) }; // SAFETY: // - We're in free_callback - so this function is safe to call. @@ -273,67 +279,59 @@ impl Object { self.make_vmap() } - /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA - /// pages for this object. + /// Acquire a revocable lease on this object's DMA scatter-gather table. /// - /// This will pin the object in memory. It is expected that `dev` should be a pointer to the - /// same [`device::Device`] which `self` belongs to, otherwise this function will return - /// `Err(EINVAL)`. - pub fn sg_table<'a>( - &'a self, - dev: &'a device::Device, - ) -> Result<&'a scatterlist::SGTable> { + /// Keep the lease for as long as any device mapping or copied DMA address uses the backing. + /// The table borrow cannot outlive the lease. The device must be this object's parent. + /// This may acquire the DMA reservation; call before locking it for a VM_BIND transaction. + pub fn sg_table<'a>(&'a self, dev: &'a device::Device) -> Result> { let parent = self.dev().as_ref(); if dev.as_raw() != parent.as_ref().as_raw() { return Err(EINVAL); } - // SAFETY: `import_attach` is set before an imported object is published and does not - // change until the object is freed. - if !unsafe { (*self.as_raw()).import_attach }.is_null() { - // The SG table of an imported object is the dma-buf attachment's mapping, which the - // exporter made for this device at import time. It belongs to the attachment, and - // `drm_gem_shmem_release()` hands it back to the exporter through - // `drm_prime_gem_destroy()`. It must not be placed in `sgt_res`, whose `SGTableMap` - // would unmap and free it as if these helpers had built it. - // - // SAFETY: `sgt` of an imported object is set before the object is published and stays - // valid and unmodified until the object is freed, which `'a` cannot outlive. + // Release and purge set the reclaim bit only while they hold this object's reservation. + // Wait for that holder instead of failing a concurrent bind with a transient EBUSY. + while let Err(e) = self.acquire_sgt_access() { + if e != EBUSY { + return Err(e); + } + drop(DmaResvGuard::new(self)); + } + // Every failure after incrementing the count drops this lease. + let lease = SGLease { obj: self.into() }; + + // SAFETY: Import attachment ownership is immutable after publication. + let sgt = if !unsafe { (*self.as_raw()).import_attach }.is_null() { + // Imports belong to the exporter. Never register them for Rust devres cleanup, + // rebuild their tables, or release their backing. + // SAFETY: The attachment table stays valid until this object is freed. let sgt = unsafe { (*self.as_raw_shmem()).sgt }; if sgt.is_null() { return Err(EINVAL); } - // SAFETY: See above. - return Ok(unsafe { scatterlist::SGTable::from_raw(sgt) }); - } - - // The borrowed reference can outlive any lock, so pin the backing permanently. - self.acquire_sgt_access(true)?; - - let sgt_res = 'out: { - // Fast path: sgt_res is already initialized - if let Some(sgt_res) = self.sgt_res.as_ref() { - break 'out sgt_res; - } - - // Slow path: Grab the lock and see if we need to initialize sgt_res. + sgt + } else { + // Register cleanup once, but cache the actual table only in C's shmem->sgt. + // Reclaim clears that pointer. No Rust reference to it survives without a lease. let _guard = self.sgt_lock.lock(); - - // If someone initialized it while we were waiting, we can exit early. - if let Some(sgt_res) = self.sgt_res.as_ref() { - break 'out sgt_res; + if self.sgt_cleanup.as_ref().is_none() { + self.sgt_cleanup.populate(Devres::new( + dev, + Ok::<_, Error>(SGTableCleanup { obj: self.into() }), + )?); } - - // If not, finish initializing and return. `populate()` cannot return false, as - // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point. - self.sgt_res - .populate(Devres::new(dev, SGTableMap::new(self))?); - - // SAFETY: We just populated sgt_res above. - unsafe { self.sgt_res.as_ref().unwrap_unchecked() } + self.sgt_cleanup.as_ref().ok_or(EIO)?.access(dev)?; + // SAFETY: The live object and lease exclude backing release. The helper locks + // the reservation and either returns the cached table or rebuilds it. + from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.as_raw_shmem()) })? }; - Ok(sgt_res.access(dev)?) + Ok(SGTable { + // SAFETY: The lease excludes release; the Bound device excludes devres revocation. + table: unsafe { scatterlist::SGTable::from_raw(sgt) }, + lease, + }) } /// Create a new shmem-backed DRM object of the given size. @@ -349,7 +347,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), - sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -360,6 +358,10 @@ impl Object { // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + // SAFETY: The object is initialized but not published. The C initializer + // reads this policy when setting up the backing mapping's GFP mask. + unsafe { (*new.as_raw_shmem()).pages_no_reclaim = config.pages_no_reclaim }; + // SAFETY: The arguments are all valid via the type invariants. to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; @@ -403,7 +405,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: None, - sgt_res: ManuallyDrop::new(SetOnce::new()), + sgt_cleanup: ManuallyDrop::new(SetOnce::new()), sgt_lock <- new_mutex!(()), sgt_access: AtomicUsize::new(0), inner <- T::new(dev, size, args), @@ -429,23 +431,18 @@ impl Object { } } - fn acquire_sgt_access(&self, permanent: bool) -> Result { + fn acquire_sgt_access(&self) -> Result { let mut state = self.sgt_access.load(Ordering::Acquire); loop { if state & SGT_RECLAIMING != 0 { return Err(EBUSY); } - let next = if permanent { - state | SGT_PERMANENT - } else { - if state & SGT_USERS == SGT_USERS { - return Err(EOVERFLOW); - } - state + 1 - }; + if state & SGT_USERS == SGT_USERS { + return Err(EOVERFLOW); + } match self.sgt_access.compare_exchange_weak( state, - next, + state + 1, Ordering::AcqRel, Ordering::Acquire, ) { @@ -466,11 +463,34 @@ impl Object { unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } } + /// Whether the object still has a cached device SG table. + /// + /// # Safety + /// + /// The caller holds the object's DMA reservation lock. + pub unsafe fn has_sg_table_locked(&self) -> bool { + // SAFETY: Reservation ownership serializes the C table pointer. + !unsafe { (*self.as_raw_shmem()).sgt }.is_null() + } + + /// Whether backing without a Rust SG lease can be reclaimed or purged. + /// + /// Also accepts backing whose SG table was already released by idle reclaim. + /// + /// # Safety + /// + /// The caller holds the object's DMA reservation lock. + pub unsafe fn unleased_pages_releasable_locked(&self) -> bool { + self.sgt_access.load(Ordering::Acquire) == 0 && + // SAFETY: The caller establishes the reservation-lock contract. + unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + } + /// Release idle backing without discarding contents. A later SG table may have different DMA /// addresses. /// - /// Refused with `EBUSY` once [`Self::sg_table`] has handed out a borrowed table, since that - /// borrow may outlive any lock. + /// Refused with `EBUSY` while any SG lease remains. The cached C table is cleared on + /// success and the next [`Self::sg_table`] rebuilds it before exposing DMA addresses. /// /// # Safety /// @@ -516,7 +536,9 @@ impl Object { /// invalidated every device mapping, and no Rust SG lease remains. pub unsafe fn purge_locked(&self) -> Result { // SAFETY: The caller's contract; the exclusive state below excludes new leases. - if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } { + if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } + || !unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + { return Err(EBUSY); } self.sgt_access @@ -706,63 +728,72 @@ where { } -/// A reference to a GEM object that is known to have a mapped [`SGTable`]. -/// -/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables -/// on GEM shmem objects are revoked on driver-unbind. -/// -/// # Invariants +/// An owned backing lease, suitable for retaining copied DMA segments. /// -/// - `self.obj` always points to a valid GEM object. -/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an -/// [`SGTable`]. -/// -/// [`SGTable`]: scatterlist::SGTable -pub struct SGTableMap { - obj: NonNull>, +/// Dropping the last lease permits reclaim after device translations have been invalidated. +/// This does not itself provide access to the table: table access also requires a Bound device. +#[must_use = "retain until all device uses of the backing have ended"] +pub struct SGLease { + obj: ARef>, } -impl Deref for SGTableMap { +impl Drop for SGLease { + fn drop(&mut self) { + let old = self.obj.sgt_access.fetch_sub(1, Ordering::Release); + debug_assert!(old & SGT_USERS != 0 && old & SGT_RECLAIMING == 0); + } +} + +/// A leased SG-table borrow. References obtained from it cannot outlive the lease. +#[must_use = "retain while using the SG table or transfer into an owned lease"] +pub struct SGTable<'a, T: DriverObject> { + table: &'a scatterlist::SGTable, + lease: SGLease, +} + +impl Deref for SGTable<'_, T> { type Target = scatterlist::SGTable; fn deref(&self) -> &Self::Target { - // SAFETY: - // - The NonNull is guaranteed to be valid via our type invariants. - // - The sgt field is guaranteed to be initialized and valid via our type invariants. - unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) } + self.table } } -impl Drop for SGTableMap { - fn drop(&mut self) { - // SAFETY: `obj` is always valid via our type invariants - let obj = unsafe { self.obj.as_ref() }; - let _lock = DmaResvGuard::new(obj); - - // SAFETY: We acquired the lock needed for calling this function above - unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) }; +impl SGTable<'_, T> { + /// Retain the backing after copying the DMA addresses, without retaining a table borrow. + pub fn into_lease(self) -> SGLease { + self.lease } } -impl SGTableMap { - fn new(obj: &Object) -> impl Init { - // INVARIANT: - // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds, - // fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized. - // SAFETY: - // - `obj` is fully initialized, making this function safe to call. - from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?; +/// Device-unbind cleanup registration for locally allocated SG tables. +/// +/// This token owns no table reference: reclaim may clear shmem->sgt and a later lease may +/// rebuild it. It is dropped before the GEM object, and is never constructed for imports. +struct SGTableCleanup { + obj: NonNull>, +} - Ok(Self { obj: obj.into() }) +impl Drop for SGTableCleanup { + fn drop(&mut self) { + // SAFETY: The registration is dropped before its GEM object. + let obj = unsafe { self.obj.as_ref() }; + let _lock = DmaResvGuard::new(obj); + + // SAFETY: Reservation ownership serializes shmem->sgt. Imports never get this token. + // Reclaim may already have freed the table; the C free helper requires a non-null one. + unsafe { + if !(*obj.as_raw_shmem()).sgt.is_null() { + bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()); + } + } } } -// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object -// it points to is guaranteed to be thread-safe. -unsafe impl Send for SGTableMap {} -// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object -// it points to is guaranteed to be thread-safe. -unsafe impl Sync for SGTableMap {} +// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. +unsafe impl Send for SGTableCleanup {} +// SAFETY: The NonNull is valid until cleanup, and its GEM object is thread-safe. +unsafe impl Sync for SGTableCleanup {} #[kunit_tests(rust_drm_gem_shmem)] mod tests { @@ -839,6 +870,14 @@ mod tests { let data = try_pin_init!(KunitData {}); let reg = faux::Registration::new(c"Kunit", None)?; let fdev = reg.as_ref(); + let dev: &device::Device = fdev.as_ref(); + // SAFETY: This new faux device is private to the fixture. Use direct DMA with a + // 64-bit mask, as the C shmem KUnit fixture does, before publishing any mappings. + unsafe { + let raw = dev.as_raw(); + (*raw).coherent_dma_mask = u64::MAX; + (*raw).dma_mask = &raw mut (*raw).coherent_dma_mask; + } let drm = UnregisteredDevice::new(fdev, data)?; Ok((reg, drm)) @@ -885,19 +924,225 @@ mod tests { // Ensure the ordering in memory is correct let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter(); for (offset, expected) in (0x20..=0x23).zip(expected) { - assert_eq!(vmap.read8(offset), expected); + assert_eq!(vmap.try_read8(offset).unwrap(), expected); } Ok(()) } - // TODO: I would love to actually test the success paths of sg_table(), but that would require - // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave - // that for someone else. + #[test] + fn sg_lease_counting() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let first = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let second = obj.sg_table(dev.as_ref().as_ref())?; + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 2); + drop(first); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 1); + drop(second); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + Ok(()) + } + + #[test] + fn release_idle_without_mappings() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held, and no device mappings or SG leases exist in this test. + unsafe { obj.release_idle_pages_locked()? }; + // SAFETY: Reservation ownership serializes these fields. + let empty = unsafe { + ( + (*obj.as_raw_shmem()).sgt.is_null(), + (*obj.as_raw_shmem()).pages.is_null(), + ) + }; + assert!(empty.0); + assert!(empty.1); + // SAFETY: Still no device mappings or leases; an already idle release succeeds too. + unsafe { obj.release_idle_pages_locked()? }; + Ok(()) + } + + #[test] + fn release_refused_while_leased() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let bind = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no actual device mappings were installed. + assert!(!unsafe { obj.sole_sg_lease_releasable_locked() }); + // SAFETY: Reservation held; no actual device mappings were installed. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + drop(bind); + // SAFETY: Reservation held; the mapping lease is now the sole owner. + assert!(unsafe { obj.sole_sg_lease_releasable_locked() }); + // SAFETY: Reservation held; the remaining lease must still exclude release. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + drop(mapping); + // SAFETY: Reservation held; no device mapping or SG lease remains. + unsafe { obj.release_idle_pages_locked()? }; + Ok(()) + } + + #[test] + fn purge_madvised_after_last_lease() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mapping = obj.sg_table(dev.as_ref().as_ref())?.into_lease(); + assert!(obj.madvise(1)); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no real device access. + assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); + drop(mapping); + // SAFETY: Reservation held; all device uses and SG leases have ended. + unsafe { obj.purge_locked()? }; + // SAFETY: Reservation ownership serializes the backing fields and advice. + let (empty_sgt, empty_pages, madv) = unsafe { + ( + (*obj.as_raw_shmem()).sgt.is_null(), + (*obj.as_raw_shmem()).pages.is_null(), + obj.madv_locked(), + ) + }; + assert!(empty_sgt); + assert!(empty_pages); + assert_eq!(madv, -1); + } + assert!(!obj.madvise(0)); + assert!(obj.sg_table(dev.as_ref().as_ref()).is_err()); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); + Ok(()) + } + + #[test] + fn sg_rebuild_after_release() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + obj.vmap::<4>()?.as_view().write32(0x12345678, 0); + for _ in 0..3 { + let table = obj.sg_table(dev.as_ref().as_ref())?; + assert!(table.iter().count() > 0); + drop(table); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; all leases have dropped, with no device mappings. + unsafe { obj.release_idle_pages_locked()? }; + // SAFETY: Reservation ownership serializes the table pointer. + assert!(unsafe { (*obj.as_raw_shmem()).sgt.is_null() }); + } + assert_eq!(obj.vmap::<4>()?.as_view().read32(0), 0x12345678); + } + // Drop with a rebuilt table exercises the surviving devres cleanup registration too. + drop(obj.sg_table(dev.as_ref().as_ref())?); + Ok(()) + } + + #[test] + fn purge_after_idle_release() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device access or SG leases remain. + unsafe { obj.release_idle_pages_locked()? }; + } + assert!(obj.madvise(1)); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device access or SG leases remain. + assert!(unsafe { obj.unleased_pages_releasable_locked() }); + // SAFETY: Reservation held; no device access or SG lease remains. + unsafe { obj.purge_locked()? }; + // SAFETY: Reservation ownership serializes the advice and table pointer. + let (madv, empty) = unsafe { (obj.madv_locked(), (*obj.as_raw_shmem()).sgt.is_null()) }; + assert_eq!(madv, -1); + assert!(empty); + Ok(()) + } + + #[test] + fn purge_refuses_kernel_vmap() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + drop(obj.sg_table(dev.as_ref().as_ref())?); + let vmap = obj.vmap::<4>()?; + assert!(obj.madvise(1)); + { + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; the vmap must prevent the backing from being freed. + assert_eq!(unsafe { obj.purge_locked() }, Err(EBUSY)); + // SAFETY: Reservation held; the vmap must exclude idle release too. + assert_eq!(unsafe { obj.release_idle_pages_locked() }, Err(EBUSY)); + } + drop(vmap); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held; no device, vmap or SG users remain. + unsafe { obj.purge_locked()? }; + Ok(()) + } + + #[test] + fn sg_reclaim_excludes_new_leases() -> Result { + let (_dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + obj.sgt_access.store(SGT_RECLAIMING, Ordering::Release); + assert_eq!(obj.acquire_sgt_access(), Err(EBUSY)); + assert_eq!(obj.sgt_access.load(Ordering::Acquire), SGT_RECLAIMING); + obj.sgt_access.store(SGT_USERS, Ordering::Release); + assert_eq!(obj.acquire_sgt_access(), Err(EOVERFLOW)); + obj.sgt_access.store(0, Ordering::Release); + Ok(()) + } + + #[test] + fn sg_import_attachment_is_not_owned() -> Result { + let (dev, drm) = create_drm_dev()?; + let obj = Object::::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?; + let mut table = MaybeUninit::::zeroed(); + let mut attachment = MaybeUninit::::zeroed(); + // SAFETY: Private fixture; neither mock is dereferenced by the import path. Both stay + // live until the lease drops and are detached before the real object destructor runs. + unsafe { + (*obj.as_raw()).import_attach = attachment.as_mut_ptr(); + (*obj.as_raw_shmem()).sgt = table.as_mut_ptr(); + } + let result = obj.sg_table(dev.as_ref().as_ref()).map(|lease| { + let no_cleanup = obj.sgt_cleanup.as_ref().is_none(); + let _guard = DmaResvGuard::new(&obj); + // SAFETY: Reservation held. Capture results before restoring the fixture, so a + // failed assertion cannot leave a mock attachment for the GEM destructor. + unsafe { + let held = obj.release_idle_pages_locked(); + drop(lease); + ( + no_cleanup, + held, + obj.release_idle_pages_locked(), + obj.purge_locked(), + (*obj.as_raw_shmem()).sgt == table.as_mut_ptr(), + ) + } + }); + // SAFETY: The mock table borrow has ended; restore the private fixture for normal free. + unsafe { + (*obj.as_raw()).import_attach = ptr::null_mut(); + (*obj.as_raw_shmem()).sgt = ptr::null_mut(); + } + let (no_cleanup, held, unleased, purge, unchanged) = result?; + assert!(no_cleanup); + assert_eq!(held, Err(EBUSY)); + assert_eq!(unleased, Err(EINVAL)); + assert_eq!(purge, Err(EBUSY)); + assert!(unchanged); + Ok(()) + } - // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it - // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device - // bound to it. + // A wrong device must fail before registering cleanup or taking any SG lease. #[test] fn fail_sg_table_on_wrong_dev() -> Result { let (_dev, drm) = create_drm_dev()?; @@ -908,10 +1153,10 @@ mod tests { assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - // If sgt_res was not initialized mistakenly with the wrong device, this should still fail. + // A failed lookup must not bind the cleanup registration to the wrong device. assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL); - // TODO: Someday, we should test that creating an sg_table here still succeeds. + assert_eq!(obj.sgt_access.load(Ordering::Acquire), 0); Ok(()) } diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index 4764d7b68f2a7..e43d5ae131c85 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -151,6 +151,59 @@ impl Mm { } } +/// A resident-set counter of an address space (`MM_*PAGES` in `struct mm_struct`). +/// +/// The counters make up the RSS that `/proc//status`, `oom_badness()` and the OOM report +/// show for a process. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum RssCounter { + /// Resident file-backed pages (`MM_FILEPAGES`). + File, + /// Resident anonymous pages (`MM_ANONPAGES`). + Anon, + /// Swapped-out anonymous pages (`MM_SWAPENTS`). + Swap, + /// Resident shared-memory pages (`MM_SHMEMPAGES`). + Shmem, +} + +impl RssCounter { + fn as_raw(self) -> core::ffi::c_int { + let member = match self { + Self::File => bindings::MM_FILEPAGES, + Self::Anon => bindings::MM_ANONPAGES, + Self::Swap => bindings::MM_SWAPENTS, + Self::Shmem => bindings::MM_SHMEMPAGES, + }; + member as core::ffi::c_int + } +} + +// The RSS counters live as long as the `mm_struct` itself (until `__mmdrop()`), so these +// methods are safe even if `mm_users` is zero. +impl Mm { + /// Adds `pages` (which may be negative) to one of the resident-set counters. + /// + /// This is how a driver attributes pages it holds on behalf of a process, but which are not + /// mapped into it, to that process. Every page added must be subtracted again before the + /// last `mmgrab` reference is dropped: `__mmdrop()` reports a non-zero counter as a bad + /// rss-counter state. + #[inline] + pub fn add_rss_counter(&self, counter: RssCounter, pages: isize) { + // SAFETY: The pointer is valid since self is a reference, and the counters are initialized + // for as long as the `mm_struct` is. + unsafe { bindings::add_mm_counter(self.as_raw(), counter.as_raw(), pages as _) }; + } + + /// Returns the exact current value of one of the resident-set counters. + #[inline] + pub fn rss_counter(&self, counter: RssCounter) -> usize { + // SAFETY: The pointer is valid since self is a reference, and the counters are initialized + // for as long as the `mm_struct` is. + unsafe { bindings::get_mm_counter_sum(self.as_raw(), counter.as_raw()) as usize } + } +} + // These methods require `mm_users` to be non-zero. impl MmWithUser { /// Obtain a reference from a raw pointer. From 3810c8bb8dda82fc049966611df523ba553f748b Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:22:53 -0500 Subject: [PATCH 10/37] Prepare the original context for carried T8122 commit 8a31fe185686 Keep the public dependency patch unchanged. The separate 7.3 adaptation restores Aurora integration after the carried series. Signed-off-by: Satya Benson --- arch/arm64/configs/asahi.config | 35 +++++++-------------------------- drivers/gpu/drm/asahi/Kconfig | 16 +++++++++++++-- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 870ff1b6b4dda..98f9a98ab9028 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -8,16 +8,9 @@ CONFIG_ARCH_APPLE=y # CONFIG_ARM64_4K_PAGES is not set CONFIG_ARM64_16K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set -CONFIG_ARM64_VA_BITS_48=y -CONFIG_ARM64_USER4K_EXPERIMENTAL=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y -# 16K pages make a page block 32 MiB, and khugepaged then sets min_free_kbytes -# to 352 MiB on an 8 GB Mac. Keep page blocks at 2 MiB as on 4K kernels. -CONFIG_PAGE_BLOCK_MAX_ORDER=7 CONFIG_ARM_APPLE_CPUIDLE=y CONFIG_ARM_APPLE_SOC_CPUFREQ=m -CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y -# CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE is not set CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m @@ -27,7 +20,7 @@ CONFIG_NVME_APPLE=m CONFIG_MMC_SDHCI_PCI=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y -CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y +CONFIG_PCIE_APPLE_PIODMA_DIAG=y CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y CONFIG_IGB=m @@ -38,7 +31,7 @@ CONFIG_INPUT_MACSMC_INPUT=m CONFIG_I2C_APPLE=m CONFIG_SPI_APPLE=m CONFIG_SPI_APPLE_QSPI=m -CONFIG_SPMI_APPLE=y +CONFIG_SPMI_APPLE=m CONFIG_PINCTRL_APPLE_GPIO=m CONFIG_GPIO_MACSMC=m CONFIG_POWER_RESET_MACSMC=m @@ -48,28 +41,19 @@ CONFIG_SENSORS_APPLE_PMP_J613=m CONFIG_THERMAL_NETLINK=y CONFIG_NVME_HWMON=y CONFIG_APPLE_WATCHDOG=y -# m1n1 hands the ADT and its own log over as phram regions (/dev/mtd*). -CONFIG_MTD=y -CONFIG_MTD_PHRAM=m -# Apple NVRAM partitions are 4 KiB aligned (J700: 0x24000). -CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y CONFIG_VIDEO_APPLE_AVD=m -CONFIG_VIDEO_APPLE_AVE=m CONFIG_VIDEO_APPLE_ISP=m -CONFIG_VIDEO_APPLE_PRORES=m -CONFIG_VIDEO_APPLE_JPEG=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m -CONFIG_DRM_ASAHI_G16=y CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m -CONFIG_DRM_APPLE_DCP_T8132=y CONFIG_DRM_APPLE_AUDIO=y CONFIG_DRM_ACCEL=y CONFIG_DRM_ACCEL_ANE=m CONFIG_DRM_SIMPLEDRM=y CONFIG_SYSFB_SIMPLEFB=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m +CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_MCA=m CONFIG_SND_SOC_APPLE_MACAUDIO=m CONFIG_SND_SOC_CS42L83=m @@ -91,14 +75,14 @@ CONFIG_USB4=m CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m -CONFIG_RTC_DRV_MACSMC=y +CONFIG_RTC_DRV_MACSMC=m CONFIG_RTC_DRV_APPLE_PMU=m CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m -CONFIG_MFD_MACSMC=y +CONFIG_APPLE_SIO_T6030=y +CONFIG_MFD_MACSMC=m CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=y -CONFIG_APPLE_DART_J713_DCP=y CONFIG_APPLE_DOCKCHANNEL=m CONFIG_SERIAL_APPLE_DOCKCHANNEL=y CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y @@ -118,20 +102,16 @@ CONFIG_APPLE_SEP=m CONFIG_APPLE_PMGR_PWRSTATE=y CONFIG_APPLE_PMP=m CONFIG_APPLE_PMP_REPORT=y -CONFIG_APPLE_PMP_THERMAL=m -CONFIG_APPLE_T8132_PLATFORM_HANDOFF=y CONFIG_IIO_AOP_SENSOR_LAS=m CONFIG_IIO_AOP_SENSOR_ALS=m -CONFIG_IIO_AOP_SENSOR_MOTION=m CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y CONFIG_PWM_APPLE=m CONFIG_APPLE_AIC=y CONFIG_PHY_APPLE_ATC=m -CONFIG_PHY_APPLE_T6040_USB2=m CONFIG_PHY_APPLE_DPTX=m CONFIG_APPLE_M1_CPU_PMU=y CONFIG_NVMEM_APPLE_EFUSES=m -CONFIG_NVMEM_APPLE_SPMI=y +CONFIG_NVMEM_APPLE_SPMI=m CONFIG_MUX_APPLE_DPXBAR=m CONFIG_TYPEC_TPS6598X=m CONFIG_RESET_APPLE_CIO=y @@ -143,4 +123,3 @@ CONFIG_RDMA_SIW=m CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_HEAPS_SYSTEM=y CONFIG_UDMABUF=y -CONFIG_DETECT_HUNG_TASK=y diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 4cfe26f28b891..ec2a588054e22 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -36,11 +36,21 @@ config DRM_ASAHI_DEBUG_ALLOCATOR Say N unless you are debugging firmware structures or porting to a new firmware version. +config DRM_ASAHI_UAT_ADOPTION + bool + help + The firmware UAT handoff adoption shared by the G15 backend and the + T8122 runtime (g15/uat.rs, g15/regs.rs). + config DRM_ASAHI_M3_T8122 bool "M3 MacBook Air GPU runtime (T8122, 14.8.3 firmware)" depends on DRM_ASAHI - # The runtime adopts the firmware's UAT through the G15 backend's handoff helpers. - depends on DRM_ASAHI_G15 + # The runtime adopts the firmware's UAT as the G15 backend does, without + # needing that backend: M3 Pro/Max keep their own setting. + # Its PMP votes call apple-pmp directly (m3_device.rs). + depends on APPLE_PMP=y || (APPLE_PMP=m && DRM_ASAHI=m) + select DRM_ASAHI_UAT_ADOPTION + select APPLE_T8122_GPU_PMP_LINK select CRYPTO_LIB_SHA256 default n help @@ -57,6 +67,8 @@ config DRM_ASAHI_G15 depends on DRM_APPLE=y || (DRM_APPLE=m && DRM_ASAHI=m) select APPLE_T6030_GPU_PMP_LINK select CRYPTO_LIB_SHA256 + select RUST_FW_LOADER_ABSTRACTIONS + select DRM_ASAHI_UAT_ADOPTION default n help Enable the native G15 backend for explicitly admitted M3 Pro/Max From 0486865371b47b9a6014b49da4f95406ddd0ee7b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:29:01 +0000 Subject: [PATCH 11/37] drm/asahi: prepare supported M3 native GPU defaults Generated-by: LLM Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> --- arch/arm64/configs/asahi.config | 2 ++ drivers/gpu/drm/asahi/Kconfig | 9 +++---- drivers/gpu/drm/asahi/m3/t8122_knobs.rs | 24 +++++++++++++++++++ drivers/gpu/drm/asahi/m3/t8122_start.rs | 31 ++++++++++++++----------- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 98f9a98ab9028..ffecb85c3beb6 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -45,6 +45,8 @@ CONFIG_VIDEO_APPLE_AVD=m CONFIG_VIDEO_APPLE_ISP=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m +CONFIG_DRM_ASAHI_M3_T8122=y +CONFIG_DRM_ASAHI_G15=y CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m CONFIG_DRM_APPLE_AUDIO=y diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index ec2a588054e22..45e907b5af86d 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -56,10 +56,11 @@ config DRM_ASAHI_M3_T8122 help Build the M3 runtime that starts the GPU of the M3 MacBook Air (J613, J615) on the 14.8.3 GPU firmware the boot loader leaves - loaded. It starts only with asahi.t8122_start=1 on the kernel - command line, after the boot loader handed the GPU over; without - it the T8122 probe is refused as before. Other GPUs are not - affected. + loaded. On the listed Air boards it starts after the boot loader + handed the GPU over and resource/firmware identity checks pass. + asahi.t8122_start=0 keeps startup disabled for recovery. Unknown + boards and firmware do not gain automatic admission. Other GPUs + are not affected. config DRM_ASAHI_G15 bool "Native G15 firmware backend (M3 Pro/Max)" diff --git a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs index b48e561f677ef..ca0f8a6d499cc 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs @@ -132,6 +132,17 @@ pub(crate) fn start(raw: &Raw) -> Start { } } +/// The shipped board policy changes only an omitted start parameter. Explicit recovery-off, +/// explicit experimental start, and malformed values retain their original meanings. +/// Hardware resources and firmware identity are admitted separately by the caller. +pub(crate) fn start_for_board(raw: &Raw, admitted_board: bool) -> Start { + if raw.start == UNSET && admitted_board { + Start::On + } else { + start(raw) + } +} + /// The GPU clock-generator IO mapping. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum ClockGen { @@ -232,6 +243,19 @@ mod tests { assert!(with(|r| r.fender = INVALID).values_given()); } + #[test] + fn automatic_start_is_board_scoped_and_explicit_values_win() { + assert_eq!(start_for_board(&Raw::NONE, true), Start::On); + assert_eq!(start_for_board(&Raw::NONE, false), Start::Off); + for admitted_board in [false, true] { + assert_eq!(start_for_board(&with(|r| r.start = 0), admitted_board), Start::Off); + assert_eq!(start_for_board(&with(|_| {}), admitted_board), Start::On); + assert_eq!(start_for_board(&with(|r| r.start = 2), admitted_board), Start::Invalid); + assert_eq!(start_for_board(&with(|r| r.start = INVALID), admitted_board), Start::Invalid); + assert_eq!(start_for_board(&with(|r| r.start = parse_number("bogus")), admitted_board), Start::Invalid); + } + } + #[test] fn defaults_are_the_plan_baseline() { let v = resolve(&with(|_| {})).unwrap(); diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs index ca9f21b56dbb6..8f03a973decb8 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_start.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs @@ -1,11 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! The T8122 (M3, G15G) start, `asahi.t8122_start=1`. +//! T8122 (M3, G15G) board-default startup and bounded experiment values. //! -//! The T8122 table (`m3_soc::T8122`) is complete, but the runtime has not passed on an M3 -//! MacBook Air yet, so the SoC is refused unless `asahi.t8122_start=1` arms the start. On a T8122 -//! whose GPU the boot loader handed over (resource admission passed), an armed start uses the -//! table's values, with a performance-state cap. Each value can be overridden for a lab A/B by +//! The T8122 table (`m3_soc::T8122`) is complete. An omitted start parameter starts +//! automatically only on its listed Air boards; explicit `asahi.t8122_start=0` +//! remains the recovery opt-out. Resource and known-firmware admission still run. +//! The same table values can be changed for a controlled experiment by //! rebooting with a parameter: //! //! | Parameter | Default | Others | @@ -19,7 +19,7 @@ //! | `asahi.t8122_pstate_cap` | `2` | 1 to 5 | //! //! The defaults are the T8122 table's values. A parameter given with a value it does not accept -//! refuses the start. Without `asahi.t8122_start=1` nothing here runs, and T8122 is refused. +//! refuses the start. Unlisted boards remain refused by the existing backend admission. //! Nothing here applies to any other SoC. use core::mem::offset_of; @@ -237,16 +237,16 @@ pub(crate) fn mmio_prot(read_only: u32, slot: usize) -> Prot { /// Decide, after resource admission, whether `soc` starts with the experiment values. /// -/// Any SoC but T8122: Ok(None), silently. T8122 without `asahi.t8122_start=1`: Ok(None), so the -/// caller refuses it as before (one extra line when experiment parameters were given). T8122 -/// with `asahi.t8122_start=1`: the experiment, after logging every value, or a refusal for a -/// parameter value it does not accept. +/// Any SoC but T8122: Ok(None). On its listed Air boards, an omitted parameter +/// selects the table values; explicit 0 disables startup, and invalid values fail +/// closed. Existing backend admission still refuses an unlisted board, even +/// with an explicit start request. pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> { if !is_t8122(soc) { return Ok(None); } let raw = crate::m3::m3_params::t8122_params(); - match knobs::start(&raw) { + match knobs::start_for_board(&raw, crate::m3::m3_board::runtime_validated_board(soc)) { Start::On => {} Start::Off => { if raw.values_given() { @@ -283,7 +283,7 @@ pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> let e = Experiment(v); dev_warn!( dev, - "M3 G15G start: armed (asahi.t8122_start=1, boot loader handoff admitted): initdata_version={:#018x} fender={:#x} clkgen={} sgx_setup={} unit_mask_a={:#x} unit_mask_b={:#x} pstate_cap={}\n", + "M3 G15G start: armed (board default or asahi.t8122_start=1, boot loader handoff admitted): initdata_version={:#018x} fender={:#x} clkgen={} sgx_setup={} unit_mask_a={:#x} unit_mask_b={:#x} pstate_cap={}\n", v.initdata_version, v.fender, clkgen_label(v.clock_gen, v.fender), @@ -352,9 +352,12 @@ pub(crate) fn refused(dev: &device::Device, experiment: Option<&Experiment>, sta } } -/// Log, on a T8122 armed with `asahi.t8122_start=1`, that resource admission refused it. +/// Log that resource admission refused a T8122 start selected by board default or flag. pub(crate) fn not_admitted(dev: &device::Device, soc: &Soc) { - if is_t8122(soc) && knobs::start(&crate::m3::m3_params::t8122_params()) == Start::On { + if is_t8122(soc) && knobs::start_for_board( + &crate::m3::m3_params::t8122_params(), + crate::m3::m3_board::runtime_validated_board(soc), + ) == Start::On { dev_err!( dev, "M3 G15G start: not armed: resource admission refused the GPU (see the lines above); the boot loader did not hand it over completely\n" From 6b505f46bda1dad14a03f202f006a66b416c1eeb Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Sun, 25 Oct 2026 20:36:35 -0400 Subject: [PATCH 12/37] drm/asahi: m3: start the GPU firmware once per boot A second start of the T8122 runtime in one boot (unbind and rebind, a module reload, or a retry after a failed start) hands the firmware freed tables. Runtime::drop and the failed-start arm free the adopted lower root once ASC has stopped, but TTBAT context 0 still names it, and the next start sets RUN before adopt_uat rewrites context 0. stop_asc only clears RUN, so there is no firmware state to restart from anyway. Refuse every start after the first one of the boot instead. The latch is claimed just before the first ASC start and is never released, and claiming it pins the module, so a reload can't forget it. An unbind and rebind now ends in a clean refusal rather than a second firmware boot, and a reboot is the only way to start the GPU again. Pinning the module also means it can't be unloaded right after an unbind while a completion work item is still queued, which was the reason for draining that work in Backend::stop. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_batch_policy.rs | 11 +++++++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 11 +++++++++++ tools/asahi/m3-batching-tests.rs | 13 +++++++++++++ 3 files changed, 35 insertions(+) diff --git a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs index 851b4e79f421e..e7a5d6ad3eecb 100644 --- a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs +++ b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs @@ -87,6 +87,17 @@ pub(crate) const fn retain_runtime(stop_failed: bool, gpu_pending: bool) -> bool stop_failed || gpu_pending } +/// One firmware start per boot. `stop_asc` only clears RUN: it doesn't reset the firmware or +/// give back a reusable RTKit or InitData state, and TTBAT context 0 still names the lower root +/// the first start adopted, which is freed once ASC has stopped. Claimed just before the first +/// ASC start and never released. +pub(crate) struct StartLatch(core::sync::atomic::AtomicBool); +impl StartLatch { + pub(crate) const fn new() -> Self { Self(core::sync::atomic::AtomicBool::new(false)) } + /// True for the first claim only. + pub(crate) fn claim(&self) -> bool { !self.0.swap(true, core::sync::atomic::Ordering::AcqRel) } +} + #[derive(Default)] pub(crate) struct PassCosts { recent: [Option<(u64, u64)>; 8] } impl PassCosts { diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 28bbea3c86889..234678e9aa2cd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -974,6 +974,8 @@ impl Inner { // Registration and Mutex initialization otherwise duplicates it on the bounded // kernel stack. ManuallyDrop retains the whole allocation on failed ASC stop. pub(crate) struct Runtime { inner: ManuallyDrop> } +/// Claimed before this boot's first ASC start. +static FIRMWARE_START: policy::StartLatch = policy::StartLatch::new(); impl Runtime { #[inline(never)] pub(crate) fn new(pdev: &platform::Device>, drm: &driver::AsahiDrmDevice, device: Device, contents: crate::m3::m3_adt_config::Contents) -> Result { @@ -995,6 +997,15 @@ impl Runtime { if crate::m3::m3_adt_config::stop_before_asc(pdev.as_ref()) { return Err(ENODEV); } + // A second start (unbind and rebind, a module reload, a retry after a failed start) + // would set RUN while TTBAT context 0 still names the freed root of this one. Unloading + // is blocked once the latch is claimed, so a reload can't forget it. + if !FIRMWARE_START.claim() { + dev_err!(pdev.as_ref(), "M3: the GPU firmware already started once this boot and can't start again; reboot to start the GPU\n"); + return Err(EBUSY); + } + // SAFETY: never released: the latch must outlive this instance of the module. + unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. let mut started=false; // The adopted UAT and its window live outside the closure: once TTBAT context 0 names diff --git a/tools/asahi/m3-batching-tests.rs b/tools/asahi/m3-batching-tests.rs index a6eb52bbf4fe7..f7db83f75862c 100644 --- a/tools/asahi/m3-batching-tests.rs +++ b/tools/asahi/m3-batching-tests.rs @@ -163,3 +163,16 @@ fn experimental_air_is_active_while_pro_retains_serial_default() { assert!(default_depth(true)<=SLOTS); assert_eq!(MAX_PACKETS,8); } + +#[test] +fn firmware_starts_once_per_boot() { + let latch=StartLatch::new(); + assert!(latch.claim()); + // Unbind and rebind, a reload or a retry: every later start is refused. + for _ in 0..3 {assert!(!latch.claim());} + // Concurrent probes: exactly one claims it. + let latch=std::sync::Arc::new(StartLatch::new()); + let claims:usize=(0..8).map(|_|{let l=latch.clone();std::thread::spawn(move||l.claim())}) + .collect::>().into_iter().map(|t|usize::from(t.join().unwrap())).sum(); + assert_eq!(claims,1); +} From 00311a5359da66cfd5c20c132b511adbf4dcc9bd Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Sun, 25 Oct 2026 20:37:29 -0400 Subject: [PATCH 13/37] drm/asahi: m3: don't quarantine VMs for packets the firmware never saw Since a failed job's VM is quarantined before its fence signals, Inner::fail_pending quarantines every pending packet's VM, including packets the firmware never saw: it finishes them without releasing their job guards. A check against the active batches can't tell them apart, because a pending packet can be partly published and fail_active drains the active batches before it calls fail_pending. Give each packet a flag that is set before the first firmware-visible change of any batch that includes it: before the producer state of a batch changes, so partial ring appends that overlapping firmware may consume count as published. fail_pending releases the guard only for a packet whose flag is unset. Everything else keeps quarantining as before, including on ECANCELED: cancellation only withdraws a packet's unpublished remainder, and its published commands may still be running. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_batch_policy.rs | 6 ++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 14 ++++++++++++-- drivers/gpu/drm/asahi/m3/m3_submit.rs | 9 ++++++++- tools/asahi/m3-batching-tests.rs | 14 ++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs index e7a5d6ad3eecb..0d8cf9056d44f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs +++ b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs @@ -83,6 +83,12 @@ pub(crate) const fn packet_retired(pending: bool, active: bool) -> bool { !pending && !active } +/// A failed packet that no batch ever included can drop its VM job guard: the firmware never saw +/// it. Any other keeps the guard, and its VM stays quarantined, until the runtime drops it. +pub(crate) const fn release_unpublished(may_be_published: bool) -> bool { + !may_be_published +} + pub(crate) const fn retain_runtime(stop_failed: bool, gpu_pending: bool) -> bool { stop_failed || gpu_pending } diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 234678e9aa2cd..aeb483cce866d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -169,9 +169,15 @@ impl Inner { } } impl Inner { - /// Fail every packet that was never published. Their fences signal with the error. + /// Fail every packet not yet fully published. Their fences signal with the error. One that no + /// batch ever included releases its VM job guard first, so its VM isn't quarantined for work + /// the firmware never saw. One that may have been published keeps it: a partly published + /// packet, or one whose batch `fail_active` has already drained from `active`. fn fail_pending(&mut self, error: Error) { - for (packet,_) in self.pending.drain_all() {packet.finish(Err(error));} + for (packet,_) in self.pending.drain_all() { + if policy::release_unpublished(packet.may_be_published()) {packet.release_guard();} + packet.finish(Err(error)); + } } /// A published batch failed or never retired: latch the fault, log it, and fail its /// packets. The packets stay retained (with their VM job guards) until teardown, because @@ -374,6 +380,9 @@ impl Inner { for (p,_) in entries { if !self.packets.iter().any(|q| Arc::ptr_eq(q,p)) {self.packets.push(p.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} } + // Before any producer state changes: an append the overlapping firmware consumes, or a + // partial publication, makes these packets visible to it. + for (p,_) in entries {p.mark_published();} let found=self.jobs.iter().position(|job| matches!((job,control), (NativeJob::Compute(_),crate::m3::m3_submit::Command::Compute(_)) | (NativeJob::Render(_),crate::m3::m3_submit::Command::Render{..}))); @@ -519,6 +528,7 @@ impl Inner { // Keep the packet before any producer mutation. No other batch is live. self.packets.clear(); self.packets.push(packet.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + packet.mark_published(); let mut render_prepared=false; let mut compute_prepared=false; let prepared=(||->Result { diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index ca16e18f10fbf..ffc57c77afcb2 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -43,6 +43,9 @@ pub(crate) struct Packet { pub(crate) wide_visibility:KVec, completion:UserFence, finish_claimed: AtomicBool, + /// Set before a batch with any of this packet's commands can change firmware-visible state, + /// and never cleared. + published: AtomicBool, /// Queue urgency (3 = UAPI realtime .. 0 = UAPI low): more urgent packets are published first. pub(crate) urgency:u8, /// Who submitted the packet, for the failure log only. @@ -78,6 +81,10 @@ impl Packet { } /// Called only after all published commands retired or before any publication. pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} + /// Called before the first firmware-visible change of a batch that includes this packet. + pub(crate) fn mark_published(&self) {self.published.store(true,Ordering::Relaxed);} + /// Whether the firmware may have seen any of this packet's commands. + pub(crate) fn may_be_published(&self)->bool {self.published.load(Ordering::Relaxed)} pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} pub(crate) fn finish(&self,result:Result) { // Retirement, cancellation and a queued scheduler timeout can race. @@ -232,7 +239,7 @@ impl queue::Queue for Queue { parser.finish().map_err(|_|EINVAL)?; let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, completion:self.fences.new_fence(0,Completion)?.into(), - finish_claimed:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), + finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; let mut job=self.entity.new_job(1,Job{shared:self.shared.clone(),packet})?; for sync in syncs.drain(0..in_sync_count) {if let Some(f)=sync.fence {job.add_dependency(f)?;}} diff --git a/tools/asahi/m3-batching-tests.rs b/tools/asahi/m3-batching-tests.rs index f7db83f75862c..2812989e51f9e 100644 --- a/tools/asahi/m3-batching-tests.rs +++ b/tools/asahi/m3-batching-tests.rs @@ -176,3 +176,17 @@ fn firmware_starts_once_per_boot() { .collect::>().into_iter().map(|t|usize::from(t.join().unwrap())).sum(); assert_eq!(claims,1); } + +#[test] +fn failed_pending_packets_quarantine_only_what_the_firmware_may_have_seen() { + // (in a batch fail_active drained, partly published and still pending, never in a batch) + struct Packet {published:bool,guard:bool} + let mut active=vec![Packet{published:true,guard:true}]; + let mut pending=vec![Packet{published:true,guard:true},Packet{published:false,guard:true}]; + // fail_active drains active before fail_pending: a check against active can't see these. + let drained:Vec<_>=active.drain(..).collect(); + for p in pending.iter_mut() {if release_unpublished(p.published) {p.guard=false;}} + assert!(drained[0].guard); + assert!(pending[0].guard,"a partly published packet keeps its guard"); + assert!(!pending[1].guard,"a never-published packet releases its guard"); +} From c297e5a440103fdac5a658f6b4e650e9e140e71d Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+chriskearney@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:16:02 +0000 Subject: [PATCH 14/37] drm/asahi: align M3 Air startup diagnostics with board defaults Signed-off-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> --- drivers/gpu/drm/asahi/m3/m3_soc.rs | 9 ++++----- drivers/gpu/drm/asahi/m3/t8122_start.rs | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_soc.rs b/drivers/gpu/drm/asahi/m3/m3_soc.rs index 7be649415cccc..4b81475612b6f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_soc.rs +++ b/drivers/gpu/drm/asahi/m3/m3_soc.rs @@ -272,10 +272,9 @@ pub(crate) fn by_chip(chip_id: u32) -> Option<&'static Soc> { /// the one the T8122 identity gate admits (`t8122_admission`: family 7, variant 2, revision 0x20, /// core slots in the first core-mask word only), with the die count of the AGX3 identification /// table (`hw::agx3::T8122`). The hardware configuration is `hw::t8122`; power configuration -/// comes from the boot loader. The rest of the runtime configuration is complete, but until the -/// runtime has passed on an M3 MacBook Air the SoC is refused, before the GPU is accessed, unless -/// `asahi.t8122_start=1` arms the start (`t8122_start`), which uses this table's values and adds a -/// performance-state cap. +/// comes from the boot loader. Startup uses the guarded `t8122_start` path, which selects this +/// table's values by default on admitted Air boards and adds a performance-state cap. Explicit +/// `asahi.t8122_start=0` disables startup; firmware, board and resource admission still apply. pub(crate) static T8122: Soc = Soc { name: "T8122", gpu_name: "G15G", @@ -326,7 +325,7 @@ pub(crate) static T8122: Soc = Soc { }, power_from_boot_loader: true, unported: &[ - "a pass on an M3 MacBook Air: until then the GPU starts only with asahi.t8122_start=1", + "M3 Air startup requires the guarded t8122_start path and its performance-state cap", ], // The performance state only. On the single-cluster G15G, reading the engine-busy or the // fault-bank registers after every job hangs the SoC within seconds at about 1000 jobs/s; diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs index 8f03a973decb8..737034dd6f8c1 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_start.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs @@ -252,7 +252,7 @@ pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> if raw.values_given() { dev_info!( dev, - "M3 G15G start: asahi.t8122_* values given without asahi.t8122_start=1; ignored\n" + "M3 G15G start: disabled by asahi.t8122_start=0; other asahi.t8122_* values ignored\n" ); } return Ok(None); From 6cdb981acb5ba801c2891861648005862cf829ec Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Sun, 25 Oct 2026 22:37:22 -0400 Subject: [PATCH 15/37] drm/asahi: m3: take the start pin with try_module_get The one-start-per-boot latch pinned the module with __module_get after claiming it. A first probe that isn't the module-init one (deferred, async or a sysfs bind) can run while an ordinary rmmod is under way: once try_stop_module has dropped the reference and marked the module GOING, __module_get revives the count unconditionally, the probe starts ASC, and the module still unloads. The next load then gets a fresh latch and starts the firmware a second time. Take the reference with try_module_get before claiming the latch, and start nothing when the module is already being removed. When another start already holds the latch, drop only the reference this probe just took; the winning claim keeps its reference, so an ordinary unload stays blocked. A forced unload, or a module init that fails after the probe started ASC, still frees the module regardless of references; neither is new here. The order lives in the batch policy (claim_start), so the host test checks it: a pin that fails claims nothing, a start that loses the claim gives back only its own reference, and the winner keeps its reference. Reported-by: Chris Kearney <303316+chriskearney@users.noreply.github.com> Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_batch_policy.rs | 25 +++++++++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 39 +++++++++++++++++---- tools/asahi/m3-batching-tests.rs | 21 +++++++++++ 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs index 0d8cf9056d44f..27295660640ae 100644 --- a/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs +++ b/drivers/gpu/drm/asahi/m3/m3_batch_policy.rs @@ -104,6 +104,31 @@ impl StartLatch { pub(crate) fn claim(&self) -> bool { !self.0.swap(true, core::sync::atomic::Ordering::AcqRel) } } +/// Why a start was refused before ASC. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum StartRefusal { + /// The module is being removed: nothing is claimed or started. + Removing, + /// This boot's start already happened: the extra module reference is dropped again. + AlreadyStarted, +} + +/// Pins the module, then claims `latch`. The pin comes first, so a probe racing an ordinary +/// rmmod fails closed instead of reviving the module's reference count. A start that loses the +/// claim drops only its own pin; the winner keeps its pin for good, so a reload can't forget the +/// latch. +pub(crate) fn claim_start(latch: &StartLatch, pin: impl FnOnce() -> bool, unpin: impl FnOnce()) + -> Result<(), StartRefusal> { + if !pin() { + return Err(StartRefusal::Removing); + } + if !latch.claim() { + unpin(); + return Err(StartRefusal::AlreadyStarted); + } + Ok(()) +} + #[derive(Default)] pub(crate) struct PassCosts { recent: [Option<(u64, u64)>; 8] } impl PassCosts { diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index aeb483cce866d..1a366d7f4b340 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -986,6 +986,24 @@ impl Inner { pub(crate) struct Runtime { inner: ManuallyDrop> } /// Claimed before this boot's first ASC start. static FIRMWARE_START: policy::StartLatch = policy::StartLatch::new(); +/// Takes a module reference, unless the module is already being removed. +#[cfg(CONFIG_MODULE_UNLOAD)] +fn pin_module() -> bool { + // SAFETY: THIS_MODULE is this module, whose code is running; try_module_get fails once + // removal has begun instead of reviving the reference count. + unsafe { kernel::bindings::try_module_get(crate::THIS_MODULE.as_ptr()) } +} +/// Without module unloading there is no removal to race. +#[cfg(not(CONFIG_MODULE_UNLOAD))] +fn pin_module() -> bool { true } +/// Drops a reference [`pin_module`] took. +#[cfg(CONFIG_MODULE_UNLOAD)] +fn unpin_module() { + // SAFETY: called only to drop a reference pin_module took. + unsafe { kernel::bindings::module_put(crate::THIS_MODULE.as_ptr()) }; +} +#[cfg(not(CONFIG_MODULE_UNLOAD))] +fn unpin_module() {} impl Runtime { #[inline(never)] pub(crate) fn new(pdev: &platform::Device>, drm: &driver::AsahiDrmDevice, device: Device, contents: crate::m3::m3_adt_config::Contents) -> Result { @@ -1008,14 +1026,21 @@ impl Runtime { return Err(ENODEV); } // A second start (unbind and rebind, a module reload, a retry after a failed start) - // would set RUN while TTBAT context 0 still names the freed root of this one. Unloading - // is blocked once the latch is claimed, so a reload can't forget it. - if !FIRMWARE_START.claim() { - dev_err!(pdev.as_ref(), "M3: the GPU firmware already started once this boot and can't start again; reboot to start the GPU\n"); - return Err(EBUSY); + // would set RUN while TTBAT context 0 still names the freed root of this one. An ordinary + // (not forced) unload is blocked once the latch is claimed, so a reload can't forget it. + // The pin comes first: a later probe can race an ordinary rmmod, and once removal has + // begun nothing starts. + match policy::claim_start(&FIRMWARE_START, pin_module, unpin_module) { + Ok(()) => {} + Err(policy::StartRefusal::Removing) => { + dev_err!(pdev.as_ref(), "M3: the module is being removed; GPU not started\n"); + return Err(ENODEV); + } + Err(policy::StartRefusal::AlreadyStarted) => { + dev_err!(pdev.as_ref(), "M3: the GPU firmware already started once this boot and can't start again; reboot to start the GPU\n"); + return Err(EBUSY); + } } - // SAFETY: never released: the latch must outlive this instance of the module. - unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. let mut started=false; // The adopted UAT and its window live outside the closure: once TTBAT context 0 names diff --git a/tools/asahi/m3-batching-tests.rs b/tools/asahi/m3-batching-tests.rs index 2812989e51f9e..d9a0a35c04dd0 100644 --- a/tools/asahi/m3-batching-tests.rs +++ b/tools/asahi/m3-batching-tests.rs @@ -190,3 +190,24 @@ fn failed_pending_packets_quarantine_only_what_the_firmware_may_have_seen() { assert!(pending[0].guard,"a partly published packet keeps its guard"); assert!(!pending[1].guard,"a never-published packet releases its guard"); } + +#[test] +fn the_start_pins_the_module_before_claiming_and_fails_closed() { + use std::cell::Cell; + let latch=StartLatch::new(); + let refs=Cell::new(1i32); // the module's base reference + let pin_ok=||{refs.set(refs.get()+1);true}; + let unpin=||refs.set(refs.get()-1); + // rmmod has begun: no pin, no claim, no start, and the reference count is untouched. + assert_eq!(claim_start(&latch,||false,unpin),Err(StartRefusal::Removing)); + assert_eq!(refs.get(),1); + // The first live start wins and keeps its pin. + assert_eq!(claim_start(&latch,pin_ok,unpin),Ok(())); + assert_eq!(refs.get(),2); + // A later start (rebind, retry) loses the claim and drops only its own pin. + assert_eq!(claim_start(&latch,pin_ok,unpin),Err(StartRefusal::AlreadyStarted)); + assert_eq!(refs.get(),2); + // A later start racing rmmod still claims nothing. + assert_eq!(claim_start(&latch,||false,||panic!("nothing to unpin")),Err(StartRefusal::Removing)); + assert_eq!(refs.get(),2); +} From ba17f5af0ba09970e4d3e33119ad19d93eaaa3f3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 28 Oct 2026 08:00:01 -0400 Subject: [PATCH 16/37] drm/asahi: m3: admit clients that ask for the parameters up to the HAL generation The M3 runtime admits a VM only for a file that asked GET_PARAMS for at least AGX3_PARAMS_SIZE bytes, and that was the size of drm_asahi_params_global. The G17 update grows the structure from 600 to 616 bytes with two optional tail fields, memory_limit and client_memory_limit, so Mesa built against the 600-byte structure would be refused VM creation with EOPNOTSUPP until it is rebuilt. The check only needs to know that the client knows the AGX3 fields. Pin the minimum at the end of gpu_hal_generation, 600 bytes, and assert that value at build time. GET_PARAMS still copies only the size the client asked for. Signed-off-by: DjDeveloperr --- drivers/gpu/drm/asahi/m3/m3_client.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs index 8fc7791e939ba..dff1af0783965 100644 --- a/drivers/gpu/drm/asahi/m3/m3_client.rs +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -5,11 +5,11 @@ //! Client admission: the M3 GPU runs AGX3 shaders and command streams. Userspace written for //! the AGX2 GPUs (G13/G14) reads the M3's parameters as those of a G14 and would submit AGX2 //! streams, which fault the GPU; the M3 backends cannot recover from a fault, so one such client -//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields at -//! the end of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks -//! GET_PARAMS for the whole structure; older userspace asks for less. On G15 a file therefore -//! has to ask for the whole structure before it can create a VM. Every path to the GPU needs a -//! VM, so a refused client can still query the parameters but never reaches the hardware. +//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields +//! of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks GET_PARAMS +//! for at least the structure up to them; older userspace asks for less. On G15 a file therefore +//! has to ask for those fields before it can create a VM. Every path to the GPU needs a VM, so a +//! refused client can still query the parameters but never reaches the hardware. use core::sync::atomic::{ AtomicU32, @@ -29,15 +29,20 @@ use crate::{ mmu, }; -/// GET_PARAMS size an AGX3-aware client asks for: the whole global parameter structure. -const AGX3_PARAMS_SIZE: u64 = core::mem::size_of::() as u64; +/// GET_PARAMS size an AGX3-aware client asks for: the global parameter structure up to the end +/// of `gpu_hal_generation`. The optional tail fields that follow it are not part of the check, +/// so growing the structure does not refuse userspace built against the shorter one. +const AGX3_PARAMS_SIZE: u64 = + (core::mem::offset_of!(uapi::drm_asahi_params_global, gpu_hal_generation) + + core::mem::size_of::()) as u64; +const _: () = assert!(AGX3_PARAMS_SIZE == 600); /// Refused clients that are logged per boot; later refusals only count. const REFUSALS_LOGGED: u32 = 16; static REFUSALS: AtomicU32 = AtomicU32::new(0); -/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for the -/// whole global parameter structure (so it knows this GPU runs AGX3 command streams), and pass +/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for at least +/// [`AGX3_PARAMS_SIZE`] bytes (so it knows this GPU runs AGX3 command streams), and pass /// VM_CREATE flags 0 (legacy HAL). `size` is the largest GET_PARAMS size the file asked for. pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) -> Result { { From 8bf9ec2ca3171eac7386853ff3372d307e4abbed Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:18:41 -0400 Subject: [PATCH 17/37] drm/asahi: accept the T8122 25G83 mailbox compatible list The 25G83 device tree advertises the AGX-specific compatible followed by asc-mailbox-v4. Both bind the same mailbox operations. Admit this exact pair while retaining all resource and identity checks. Exercise the production rejection predicate with public-DTS fixtures and resource mutations; allow optional local DT captures for reproduction. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_soc.rs | 1 + tools/asahi/run-host-tests.py | 1 + tools/asahi/test-m3-mailbox.py | 90 ++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 tools/asahi/test-m3-mailbox.py diff --git a/drivers/gpu/drm/asahi/m3/m3_soc.rs b/drivers/gpu/drm/asahi/m3/m3_soc.rs index 4b81475612b6f..7500822c4124c 100644 --- a/drivers/gpu/drm/asahi/m3/m3_soc.rs +++ b/drivers/gpu/drm/asahi/m3/m3_soc.rs @@ -291,6 +291,7 @@ pub(crate) static T8122: Soc = Soc { cores_per_cluster: 10, mailbox_compatibles: &[ b"apple,t8122-agx-asc-mailbox\0", + b"apple,t8122-agx-asc-mailbox\0apple,asc-mailbox-v4\0", b"apple,t8122-asc-mailbox\0apple,asc-mailbox-v4\0", ], mailbox_interrupts: &[0, 723, 4, 0, 724, 4, 0, 725, 4, 0, 726, 4], diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 5c5ca93c68594..c4c6e21383fbf 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,6 +53,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'test-m3-mailbox.py': [([], PASS)], 'aop-als-channel-tests.py': [([], PASS)], 'mtp-pm-tests.py': [(out(), PASS), (out() + ['--old-transport', '4db3a5a22c43'], PASS)], diff --git a/tools/asahi/test-m3-mailbox.py b/tools/asahi/test-m3-mailbox.py new file mode 100644 index 0000000000000..2320d17f10f6f --- /dev/null +++ b/tools/asahi/test-m3-mailbox.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production T8122 mailbox predicate with captured DTB properties. + +Usage: python3 tools/asahi/test-m3-mailbox.py [runtime.dtb daily.dtb] +Requires dtc's fdtget. No target access or MMIO. +""" +import hashlib +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +board = (ROOT / 'drivers/gpu/drm/asahi/m3/m3_board.rs').read_text() +soc = (ROOT / 'drivers/gpu/drm/asahi/m3/m3_soc.rs').read_text() +t8122 = soc[soc.index('pub(crate) static T8122:'):] +compatibles = re.search(r'mailbox_compatibles: &(\[.*?\]),', t8122, re.S)[1] +irqs = re.search(r'mailbox_interrupts: &(\[.*?\]),', t8122, re.S)[1] +base = re.search(r'mailbox: (0x[0-9a-fA-F_]+),', t8122)[1] +regfn = re.search(r'fn expected_mailbox_reg\(.*?\n}', board, re.S)[0] +predicate = board[board.index(' if !soc.mailbox_compatibles'):] +predicate = predicate[:predicate.index('\n {')].strip()[3:] + + +def prop(dtb, node, name, kind): + result = subprocess.check_output(['fdtget', '-t', kind, str(dtb), node, name], text=True) + return [int(v, 16) for v in result.split()] + + +def vec(values): + return 'vec![' + ','.join(map(str, values)) + ']' + + +fixtures = [] +for argument in sys.argv[1:]: + dtb = Path(argument) + gpu = '/soc/gpu@292400000' + # Both retained captures use this mailbox path; verify the phandle link, + # not its capture-specific numeric value. + mbox = '/soc/mbox@292408000' + if prop(dtb, gpu, 'mboxes', 'x') != prop(dtb, mbox, 'phandle', 'x'): + raise RuntimeError('GPU mailbox phandle mismatch') + values = [prop(dtb, mbox, name, kind) for name, kind in + [('compatible', 'bx'), ('reg', 'x'), ('interrupt-names', 'bx'), + ('interrupts', 'x'), ('#mbox-cells', 'x')]] + if len(values[4]) != 1: + raise RuntimeError('bad #mbox-cells length') + fixtures.append('check(' + ','.join(vec(v) for v in values[:4]) + ',' + str(values[4][0]) + ');') + print(dtb.name, hashlib.sha256(dtb.read_bytes()).hexdigest()) +if not sys.argv[1:]: + # Public t8122-j613-dcp-25g83.dtsi and t8122-gpu.dtsi resource contract. + names = list(b'send-empty\0send-not-empty\0recv-empty\0recv-not-empty\0') + for compat in (b'apple,t8122-agx-asc-mailbox\0apple,asc-mailbox-v4\0', + b'apple,t8122-asc-mailbox\0apple,asc-mailbox-v4\0'): + values = [list(compat), [2, 0x92408000, 0, 0x4000], names, + [0,723,4,0,724,4,0,725,4,0,726,4]] + fixtures.append('check(' + ','.join(vec(v) for v in values) + ',0);') +if len(fixtures) != 2: + raise SystemExit('supply runtime.dtb and daily.dtb') + +source = ''' +struct Windows { mailbox: u64 } +struct Soc { mailbox_compatibles: &'static [&'static [u8]], mailbox_interrupts: &'static [u32], windows: Windows } +''' + regfn + ''' +fn accepts(soc: &Soc, compat: &Vec, mbox_reg: &Vec, irq_names: &Vec, irqs: &Vec, cells: u32) -> bool { + !(''' + predicate + ''') +} +fn check(compat: Vec, reg: Vec, names: Vec, irqs: Vec, cells: u32) { + let soc = Soc { mailbox_compatibles: &''' + compatibles + ''', mailbox_interrupts: &''' + irqs + ''', windows: Windows { mailbox: ''' + base + ''' } }; + let ok = |c: &Vec, r: &Vec, n: &Vec, i: &Vec, x| accepts(&soc,c,r,n,i,x); + assert!(ok(&compat,®,&names,&irqs,cells)); + for i in 0..reg.len() { let mut v=reg.clone(); v[i]^=1; assert!(!ok(&compat,&v,&names,&irqs,cells)); } + for i in 0..irqs.len() { let mut v=irqs.clone(); v[i]^=1; assert!(!ok(&compat,®,&names,&v,cells)); } + assert!(!ok(&compat,®,&names,&irqs,1)); + let mut v=names.clone(); v[0]^=1; assert!(!ok(&compat,®,&v,&irqs,cells)); + for v in [b"apple,asc-mailbox-v4\\0".to_vec(), b"apple,t8122-agx-asc-mailbox\\0unknown\\0".to_vec(), b"apple,t6031-agx-asc-mailbox\\0".to_vec()] { assert!(!ok(&v,®,&names,&irqs,cells)); } + for v in soc.mailbox_compatibles { assert!(ok(&v.to_vec(),®,&names,&irqs,cells)); } +} +fn main() { +''' + '\n'.join(fixtures) + '\nprintln!("mailbox: both captures, all exact supported lists and resource/identity mutation controls PASS");\n}\n' +with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) + (path / 'test.rs').write_text(source) + subprocess.run([os.environ.get('RUSTC', 'rustc'), '--edition=2021', str(path / 'test.rs'), '-o', str(path / 'test')], check=True) + subprocess.run([str(path / 'test')], check=True) +print('m3_board.rs', hashlib.sha256(board.encode()).hexdigest()) +print('m3_soc.rs', hashlib.sha256(soc.encode()).hexdigest()) From dfb9d8b576a0b6c0364990c4f3487737413524bc Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:38:58 -0400 Subject: [PATCH 18/37] drm/asahi: describe the T8122 extended firmware profile and owned IO records Separate firmware identity, HAL metadata and VM bounds from physical G15G stream selection. Add checked root and owned IO descriptor encoders. Introduce slot 28 at SGX+0xd50000 with a production-writer regression; the shifted T8132 bank is never introduced. Keep VM padding zero. Signed-off-by: Satya Benson --- MAINTAINERS | 2 + drivers/gpu/drm/apple/j613_dcp_activate.c | 9 +- drivers/gpu/drm/asahi/m3/m3_client.rs | 17 +- drivers/gpu/drm/asahi/m3/m3_drm.rs | 20 +- drivers/gpu/drm/asahi/m3/m3_firmware.rs | 146 +++- drivers/gpu/drm/asahi/m3/m3_init_26.rs | 275 +++++++ drivers/gpu/drm/asahi/m3/m3_init_layout.rs | 139 +++- drivers/gpu/drm/asahi/m3/m3_mmio_26.rs | 265 +++++++ drivers/gpu/drm/asahi/m3/m3_profile.rs | 69 ++ drivers/gpu/drm/asahi/m3/mod.rs | 4 + drivers/gpu/drm/asahi/m3/t8122_admission.rs | 49 +- tools/asahi/m3-firmware-tests.rs | 14 + tools/asahi/m3-runtime-tests.rs | 9 + tools/asahi/m3-uapi-fixture.c | 363 +++++++++ tools/asahi/m3-uapi-tests.rs | 832 ++++++++++++++++++++ tools/asahi/run-host-tests.py | 3 + tools/asahi/test-j613-dcp-power.py | 57 ++ tools/asahi/test-m3-mailbox.py | 11 + tools/asahi/test-m3-uapi.py | 73 ++ 19 files changed, 2333 insertions(+), 24 deletions(-) create mode 100644 drivers/gpu/drm/asahi/m3/m3_init_26.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_mmio_26.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_profile.rs create mode 100644 tools/asahi/m3-firmware-tests.rs create mode 100644 tools/asahi/m3-uapi-fixture.c create mode 100644 tools/asahi/m3-uapi-tests.rs create mode 100644 tools/asahi/test-j613-dcp-power.py create mode 100755 tools/asahi/test-m3-uapi.py diff --git a/MAINTAINERS b/MAINTAINERS index e0a0070243ae6..dbe76f9d15eaf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2789,8 +2789,10 @@ F: tools/asahi/apple-pwm-pm-tests.py F: drivers/reset/reset-apple-cio.c F: drivers/rtc/rtc-macsmc.c F: drivers/soc/apple/* +F: tools/asahi/m3-uapi-* F: tools/asahi/pmp-bootargs-tests.rs F: drivers/spi/spi-apple-test.c +F: tools/asahi/test-m3-uapi.py F: drivers/spi/spi-apple.c F: drivers/spmi/spmi-apple-controller.c F: drivers/thermal/apple-pmp* diff --git a/drivers/gpu/drm/apple/j613_dcp_activate.c b/drivers/gpu/drm/apple/j613_dcp_activate.c index 6e20a5afc914c..6d0a262069469 100644 --- a/drivers/gpu/drm/apple/j613_dcp_activate.c +++ b/drivers/gpu/drm/apple/j613_dcp_activate.c @@ -40,6 +40,13 @@ bool apple_t8122_25g83_board(void) return ok; } +/* Match the PMGR provider: automatic domains can be clock-gated while active. */ +static bool j613_power_state_active(u32 state) +{ + return FIELD_GET(GENMASK(7, 4), state) == 15 || + ((state & BIT(28)) && FIELD_GET(GENMASK(3, 0), state) == 15); +} + /* These are DT domain nodes, carrying the PMGR offsets, not copied registers. */ static int j613_power_ready(void) { @@ -82,7 +89,7 @@ static int j613_power_ready(void) return -ENOMEM; state = readl(pmgr + offset); iounmap(pmgr); - if (FIELD_GET(GENMASK(7, 4), state) != 15) + if (!j613_power_state_active(state)) return -EBUSY; } return 0; diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs index dff1af0783965..c1a07d61120d1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_client.rs +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -35,7 +35,7 @@ use crate::{ const AGX3_PARAMS_SIZE: u64 = (core::mem::offset_of!(uapi::drm_asahi_params_global, gpu_hal_generation) + core::mem::size_of::()) as u64; -const _: () = assert!(AGX3_PARAMS_SIZE == 600); +const _: () = assert!(AGX3_PARAMS_SIZE == super::m3_profile::PARAMS_REQUIRED); /// Refused clients that are logged per boot; later refusals only count. const REFUSALS_LOGGED: u32 = 16; @@ -45,10 +45,19 @@ static REFUSALS: AtomicU32 = AtomicU32::new(0); /// [`AGX3_PARAMS_SIZE`] bytes (so it knows this GPU runs AGX3 command streams), and pass /// VM_CREATE flags 0 (legacy HAL). `size` is the largest GET_PARAMS size the file asked for. pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) -> Result { + admit_vm_for(device, super::m3_profile::Profile::Legacy, size, flags) +} + +pub(crate) fn admit_vm_for( + device: &kernel::device::Device, + profile: super::m3_profile::Profile, + size: u64, + flags: u32, +) -> Result { { // The positive acknowledgement of the command ABI (7.1.12 `g16_profile::client_admitted`): // GET_PARAMS coverage, and VM_CREATE flags 0 on the legacy-HAL M3 runtime. - if size >= AGX3_PARAMS_SIZE && flags == 0 { + if profile.vm_acknowledged(size, flags) { return Ok(()); } let refusals = REFUSALS.fetch_add(1, Ordering::Relaxed).saturating_add(1); @@ -64,11 +73,13 @@ pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) - }; dev_info!( device, - "M3: refusing a VM for {}[{}]: it asked GET_PARAMS for {} of the {} parameter bytes, so it cannot know this GPU runs AGX3 command streams; M3-aware userspace is needed{}\n", + "M3: refusing a VM for {}[{}]: GET_PARAMS bytes={}/{} HAL={} flags={:#x}{}\n", name, task.group_leader().pid(), size, AGX3_PARAMS_SIZE, + profile.hal(), + flags, last ); } diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index dc8f1a074e809..796abc7793f90 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -86,6 +86,7 @@ pub(crate) struct Backend { core_mask: u64, max_frequency_khz: u32, soc: &'static crate::m3::m3_soc::Soc, + profile: super::m3_profile::Profile, _progress: Pin>>, _activity: Pin>>, _memory: Pin>>, @@ -125,6 +126,7 @@ impl Backend { let firmware = crate::m3::m3_firmware::identify_loaded(pdev, soc, resources, experiment) .inspect_err(refused("the firmware identity"))?; dev_info!(pdev.as_ref(), "M3: firmware identity accepted\n"); + let profile = firmware.profile(); // InitData source and contents, before any GPU register access. let contents = crate::m3::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment) .inspect_err(refused("the InitData contents"))?; @@ -160,7 +162,7 @@ impl Backend { let completion_work = crate::m3::m3_completion::Completion::new(shared.clone())?; // From here on every failure drops the backend, which stops the runtime. let backend = Arc::new(Self { shared, health, ids: gpu::SequenceIDs::default(), - scheduler, core_mask, max_frequency_khz, soc, + scheduler, core_mask, max_frequency_khz, soc, profile, _progress: progress, _activity: activity, _memory: memory, _timing: timing, _completion: completion, _overlap: overlap }, GFP_KERNEL)?; // The completion worker services this runtime from now on; the RTKit callbacks and @@ -208,7 +210,9 @@ impl gpu::Gpu for Backend { } params.max_frequency_khz = self.max_frequency_khz; params.usc_generation = 3; - params.gpu_hal_generation = hw::GpuHalGeneration::Legacy as u32; + params.gpu_hal_generation = self.profile.hal(); + params.max_commands_per_submission = super::agx_uapi::UAPI_MAX_HARDWARE_COMMANDS; + params.max_attachments = super::agx_uapi::UAPI_MAX_ATTACHMENTS as u32; Ok(()) } fn supports_vm_status(&self) -> bool { true } @@ -217,10 +221,12 @@ impl gpu::Gpu for Backend { fn feature_flags(&self) -> u64 { let mut flags = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SCHEDULED_QUEUES as u64; - if self.soc.features.fragment_dependency && crate::m3::m3_params::early_tiling() { + if self.profile == super::m3_profile::Profile::Legacy + && self.soc.features.fragment_dependency && crate::m3::m3_params::early_tiling() { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY as u64; } - if self.soc.features.compute_wide_visibility { + if self.profile == super::m3_profile::Profile::Legacy + && self.soc.features.compute_wide_visibility { flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY as u64; } flags @@ -233,17 +239,17 @@ impl gpu::Gpu for Backend { } fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } fn user_range(&self) -> Range { - mmu::UAT_PGSZ as u64..crate::m3::initdata::RUNTIME_UNKNOWN_PAGE + mmu::UAT_PGSZ as u64..self.profile.user_end() } /// The dummy page sits where InitData tells the firmware it is (unk_30), as on 7.1.12; the /// 42-bit geometry's own unk_page() is a different address. fn unknown_page(&self) -> u64 { - crate::m3::initdata::RUNTIME_UNKNOWN_PAGE + self.profile.user_end() } fn tracked_submission(&self) -> bool { true } /// Only M3-aware userspace may create a VM (7.1.12's per-file client gate). fn admit_vm(&self, dev: &kernel::device::Device, params_size: u64, flags: u32) -> Result { - crate::m3::m3_client::admit_vm(dev, params_size, flags) + crate::m3::m3_client::admit_vm_for(dev, self.profile, params_size, flags) } fn new_vm(&self, range: Range) -> Result { Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.new_vm(self.ids.vm.next(), range) diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index 0d2835174c677..c9aa45cc02864 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -62,6 +62,19 @@ pub(crate) static KNOWN_IMAGES_T8122: [KnownImage; 1] = [KnownImage { initdata_magic: Some(G15_V14_8_3_INITDATA), }]; +/// Image record for the extended format. The SoC's runtime selection does not use it yet. +pub(crate) static T8122_IMAGES_26: [KnownImage; 1] = [KnownImage { + name: "T8122 C0 firmware-compat 26.6.2", + uuid: None, + stkg_sha256: Some([ + 0x3a, 0x6b, 0x8b, 0x5e, 0x83, 0x38, 0x4d, 0x5c, + 0xc1, 0x92, 0x9d, 0x91, 0xb6, 0x36, 0x40, 0xfe, + 0xe8, 0xf5, 0x52, 0xb5, 0x10, 0x8c, 0x4e, 0xa6, + 0x20, 0xbe, 0xdb, 0x7b, 0xbc, 0xa0, 0x31, 0x64, + ]), + initdata_magic: Some(0x0c89_c357_8392_04b8), +}]; + /// The same T8122 image for the start experiment (`t8122_start`, only with /// `asahi.t8122_start=1`), where `asahi.t8122_initdata_version` may replace the version. pub(crate) static KNOWN_IMAGES_T8122_EXPERIMENT: [KnownImage; 1] = [KnownImage { @@ -117,6 +130,10 @@ pub(crate) struct Layout { /// Offset of the 8-byte tag that precedes the 8 per-boot bytes zeroed before hashing. pub(crate) entropy_tag_offset: usize, pub(crate) entropy_tag: [u8; 8], + /// An additional independent boot value, when required by this image. + extra_boot_value: Option<(usize, [u8; 8])>, + /// Expected boot-table location and byte count in the image header. + boot_table: Option<(u32, u32)>, /// The firmware version, for logs. pub(crate) version: &'static str, } @@ -127,6 +144,8 @@ pub(crate) static T6030_LAYOUT: Layout = Layout { vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0005_c000], entropy_tag_offset: 0x58e88, entropy_tag: *b"GKTS\x08\x00\x00\x00", + extra_boot_value: None, + boot_table: None, version: "RTKit-2419.140.12.release", }; @@ -137,23 +156,73 @@ pub(crate) static T8122_LAYOUT: Layout = Layout { vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], entropy_tag_offset: 0x5d54a, entropy_tag: *b"GKTS\x08\x00\x00\x00", + extra_boot_value: None, + boot_table: None, version: "RTKit-2419.140.12.release", }; pub(crate) const TEXT_SHA256: [u8; 32] = [ 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; +/// T8122 C0, firmware-compat 26.6.2. Selection remains separate from runtime admission. +pub(crate) static T8122_LAYOUT_26: Layout = Layout { + text_size: 393216, + data_size: 1228800, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_0000], + entropy_tag_offset: 380492, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + extra_boot_value: Some((380719, *b"ECAP\x08\x00\x00\x00")), + boot_table: Some((380492, 561)), + version: "RTKit-3255.160.4.release", +}; + fn normalize_boot_entropy(layout: &Layout, text: &mut [u8]) -> bool { - let tag = layout.entropy_tag_offset; if text.len() != layout.text_size as usize { return false; } - let Some(end) = tag.checked_add(16) else { return false; }; - let Some(entropy) = text.get_mut(tag..end) else { return false; }; - if entropy[..8] != layout.entropy_tag { - return false; + if let Some((offset, count)) = layout.boot_table { + let Some(header) = text.get(0x22c..0x234) else { + return false; + }; + if header[..4] != offset.to_le_bytes() || header[4..] != count.to_le_bytes() { + return false; + } + let Some(end) = (offset as usize).checked_add(count as usize) else { + return false; + }; + if text.get(offset as usize..end).is_none() { + return false; + } + } + let fields = [ + Some((layout.entropy_tag_offset, layout.entropy_tag)), + layout.extra_boot_value, + ]; + // Validate every field before touching even the first boot value. + for (offset, tag) in fields.into_iter().flatten() { + let Some(end) = offset.checked_add(16) else { + return false; + }; + let Some(field) = text.get(offset..end) else { + return false; + }; + if field[..8] != tag { + return false; + } + if let Some((base, count)) = layout.boot_table { + if offset < base as usize || end > base as usize + count as usize { + return false; + } + } + } + if let Some((other, _)) = layout.extra_boot_value { + if other < layout.entropy_tag_offset + 16 && layout.entropy_tag_offset < other + 16 { + return false; + } + } + for (offset, _) in fields.into_iter().flatten() { + text[offset + 8..offset + 16].fill(0); } - entropy[8..].fill(0); true } @@ -167,6 +236,14 @@ pub(crate) struct Firmware { } impl Firmware { + pub(crate) fn profile(&self) -> super::m3_profile::Profile { + if core::ptr::eq(self.layout, &T8122_LAYOUT_26) { + super::m3_profile::Profile::Extended + } else { + super::m3_profile::Profile::Legacy + } + } + pub(crate) const fn version(&self) -> &'static str { self.layout.version } @@ -393,4 +470,61 @@ mod tests { assert!(!normalize_boot_entropy(&layout, &mut text)); assert_eq!(text, vec![0xa5; 15]); } + + fn image_26() -> Vec { + let mut image = vec![0x5a; 393216]; + image[0x22c..0x230].copy_from_slice(&380492u32.to_le_bytes()); + image[0x230..0x234].copy_from_slice(&561u32.to_le_bytes()); + image[380492..380500].copy_from_slice(b"GKTS\x08\x00\x00\x00"); + image[380719..380727].copy_from_slice(b"ECAP\x08\x00\x00\x00"); + image + } + + #[test] + fn newer_image_masks_only_both_boot_values() { + let mut image = image_26(); + let before = image.clone(); + assert!(normalize_boot_entropy(&T8122_LAYOUT_26, &mut image)); + for index in 0..image.len() { + let expected = if (380500..380508).contains(&index) || (380727..380735).contains(&index) + { + 0 + } else { + before[index] + }; + assert_eq!(image[index], expected, "byte {index}"); + } + } + + #[test] + fn invalid_second_field_or_table_does_not_mask_first_value() { + for index in [0x22c, 0x230, 380492, 380496, 380719, 380723] { + let mut image = image_26(); + image[index] ^= 1; + let before = image.clone(); + assert!(!normalize_boot_entropy(&T8122_LAYOUT_26, &mut image)); + assert_eq!(image, before); + } + let mut image = image_26(); + image.pop(); + let before = image.clone(); + assert!(!normalize_boot_entropy(&T8122_LAYOUT_26, &mut image)); + assert_eq!(image, before); + } + + #[test] + fn extended_record_cannot_replace_legacy_identity() { + let newer = &T8122_IMAGES_26[0]; + let digest = newer.stkg_sha256.unwrap(); + assert!(newer.accepts_runtime(None, &digest)); + assert_eq!(newer.initdata_magic, Some(0x0c89_c357_8392_04b8)); + for index in 0..digest.len() { + let mut wrong = digest; + wrong[index] ^= 1; + assert!(!newer.accepts_runtime(None, &wrong)); + } + assert!(!KNOWN_IMAGES_T8122[0].accepts_runtime(None, &digest)); + assert!(!newer.accepts_runtime(KNOWN_IMAGES_T8122[0].uuid, &[0; 32])); + } + } diff --git a/drivers/gpu/drm/asahi/m3/m3_init_26.rs b/drivers/gpu/drm/asahi/m3/m3_init_26.rs new file mode 100644 index 0000000000000..f08f16200b070 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_init_26.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Allocation and pointer records for the extended M3 initialization format. +//! Owners supply mapped regions; these builders never allocate or publish DMA. + +use super::m3_init_layout::{Error, Region, Root}; +use super::m3_init_storage::{self as storage, Allocation, Initial}; + +pub(crate) const COUNT: usize = 56; +pub(crate) const DYNAMIC_STATE: usize = 49; +pub(crate) const PERFORMANCE_CONTROL: usize = 50; +pub(crate) const RECOVERY_ACTIVITY: usize = 51; +pub(crate) const SENSOR_VALUE: usize = 52; +pub(crate) const RETENTION_STATE: usize = 53; +pub(crate) const GLOBAL_POOL: usize = 54; +pub(crate) const GLOBAL_PARAMETERS: usize = 55; +pub(crate) const RUNTIME_SIZE: usize = 0x4b8; +pub(crate) const LOG_THREADS: usize = 9; +pub(crate) const RX_BYTES: usize = 0x48; +pub(crate) const LOG_PAYLOAD_BYTES: usize = 0xd8; + +/// Extend the existing M3 owner indices instead of introducing another owner graph. +pub(crate) fn allocation(index: usize) -> Result { + let size = match index { + 0..=23 if index % 2 == 0 => 0x30, + 0..=23 => 0x1800, + 24 | 26 | 30 | 32 => 0x30, + 25 => 0x4000, + 27 | 33 => 0x4800, + 28 => LOG_THREADS * 0x30, + 29 => LOG_THREADS * 256 * RX_BYTES, + 31 => 512 * RX_BYTES, + 34 => 0x40, + 35 => 0x3fc0, + 36 => 0xc8, + 37 => 0x4000, + 38 => RUNTIME_SIZE, + 39 => 0x2658, + 40 => 0x88, + 41 => LOG_THREADS * 256 * LOG_PAYLOAD_BYTES, + 42 => 0xc18, + 43 => 0x1248, + 44 => 0xe10, + 45 => 0xe48, + 46 => 0x670, + 47 => 0xeaf0, + 48 => 0x20, + DYNAMIC_STATE | RETENTION_STATE | GLOBAL_POOL | GLOBAL_PARAMETERS => 0x4000, + PERFORMANCE_CONTROL => 0x6578, + RECOVERY_ACTIVITY => 0x60, + SENSOR_VALUE => 4, + _ => return Err(Error::Bounds), + }; + // Separate page-aligned owners, with room for the largest receive allocation. + // This namespace is disjoint from the existing fixed work and timestamp arenas. + Ok(Allocation { + address: 0xffff_fc21_0000_0000 + (index as u64) * 0x10_0000, + size, + gpu_shared: matches!(index, GLOBAL_POOL | GLOBAL_PARAMETERS), + initial: Initial::Zero, + }) +} + +pub(crate) fn cached(index: usize) -> bool { + matches!(index, 38 | 39 | 42 | 43 | 44 | PERFORMANCE_CONTROL) +} + +pub(crate) struct PointerGraph<'a> { + pub(crate) owners: &'a [Region; COUNT], + pub(crate) scratch: Region, + pub(crate) pool_alias: u64, + pub(crate) parameter_alias: u64, +} + +impl PointerGraph<'_> { + /// Check the complete graph, including padded mapping ownership, before encoding it. + fn addresses(&self) -> Result<[u64; COUNT], Error> { + let mut addresses = [0; COUNT]; + for (index, owner) in self.owners.iter().enumerate() { + let size = allocation(index)?.size; + addresses[index] = owner.at(0, size)?; + if addresses[index] & 0x3fff != 0 + || addresses[index] < 0xffff_fc20_0000_0000 + || addresses[index] >= 0xffff_fc2f_fc00_0000 + { + return Err(Error::Address); + } + let mapped = (size + 0x3fff) & !0x3fff; + let end = addresses[index] + .checked_add(mapped as u64) + .ok_or(Error::Address)?; + if end > 0xffff_fc2f_fc00_0000 { + return Err(Error::Address); + } + for prior in 0..index { + let prior_size = (allocation(prior)?.size + 0x3fff) & !0x3fff; + if addresses[index] < addresses[prior] + prior_size as u64 + && addresses[prior] < addresses[index] + mapped as u64 + { + return Err(Error::Address); + } + } + } + let scratch = self.scratch.at(0, 0x8000)?; + let scratch_end = scratch.checked_add(0x8000).ok_or(Error::Address)?; + if scratch & 0x3fff != 0 || scratch_end > 0xffff_fc2f_fc00_0000 { + return Err(Error::Address); + } + for (index, address) in addresses.iter().enumerate() { + let bytes = (allocation(index)?.size + 0x3fff) & !0x3fff; + if scratch < *address + bytes as u64 && *address < scratch_end { + return Err(Error::Address); + } + } + for alias in [self.pool_alias, self.parameter_alias] { + if alias & 0x3fff != 0 || !(0x70_0000_0000..0x70_0200_0000).contains(&alias) { + return Err(Error::Address); + } + } + if self.pool_alias == self.parameter_alias { + return Err(Error::Address); + } + Ok(addresses) + } + + pub(crate) fn root(&self, output: &mut [u8]) -> Result<(), Error> { + self.addresses()?; + Root { + region_a: self.owners[storage::REGION_A], + runtime: self.owners[storage::RUNTIME_POINTERS], + globals: self.owners[storage::GLOBALS], + new_a: self.owners[storage::RUNTIME_FLAGS], + control: self.owners[storage::CONTROL_REGION], + power: self.owners[storage::GLOBALS_POWER], + } + .encode_26(output, self.owners[DYNAMIC_STATE]) + } + + pub(crate) fn runtime(&self, output: &mut [u8]) -> Result<(), Error> { + let addresses = self.addresses()?; + let scratch = self.scratch.at(0, 0x8000)?; + let bytes = output.get_mut(..RUNTIME_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + let pointer = |bytes: &mut [u8], offset: usize, value: u64| { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + }; + pointer(bytes, 0x18, scratch); + pointer(bytes, 0x2d0, self.parameter_alias); + pointer(bytes, 0x2e0, self.pool_alias); + bytes[0x250..0x254].copy_from_slice(&1u32.to_le_bytes()); + bytes[0x2f8..0x2fc].copy_from_slice(&4u32.to_le_bytes()); + for (offset, owner) in [ + (0, storage::HARDWARE_DATA), + (0x10, storage::UNKNOWN_PAIR), + (0x1c0, 26), + (0x1c8, 27), + (0x1d0, 28), + (0x1d8, 29), + (0x1e0, 30), + (0x1e8, 31), + (0x1f0, 32), + (0x1f8, 33), + (0x200, storage::FWLOG_PAYLOAD), + (0x254, storage::UNKNOWN_C0), + (0x25c, storage::UNKNOWN_C1), + (0x264, storage::UNKNOWN_C3), + (0x26c, RECOVERY_ACTIVITY), + (0x2d8, GLOBAL_PARAMETERS), + (0x2e8, GLOBAL_POOL), + (0x469, PERFORMANCE_CONTROL), + ] { + pointer(bytes, offset, addresses[owner]); + } + for channel in 0..13 { + let offset = 0x20 + channel * 0x20; + let state = addresses[channel * 2]; + pointer(bytes, offset, state); + pointer(bytes, offset + 8, state + 0x10); + pointer(bytes, offset + 16, state + 0x20); + pointer(bytes, offset + 24, addresses[channel * 2 + 1]); + } + Ok(()) + } + + pub(crate) fn firmware_control(&self, output: &mut [u8]) -> Result<(), Error> { + let addresses = self.addresses()?; + let bytes = output + .get_mut(..allocation(storage::CONTROL_REGION)?.size) + .ok_or(Error::Size)?; + bytes.fill(0); + bytes[0x4f88..0x4f90].copy_from_slice(&addresses[storage::FW_CONTROL_STATE].to_le_bytes()); + bytes[0x4f90..0x4f98].copy_from_slice(&addresses[storage::FW_CONTROL_RING].to_le_bytes()); + bytes[0x4fd4..0x4fd8].copy_from_slice(&1u32.to_le_bytes()); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn owners(delta: u64) -> [Region; COUNT] { + core::array::from_fn(|index| { + let item = allocation(index).unwrap(); + Region::new(item.address + delta, item.size).unwrap() + }) + } + + fn graph(owners: &[Region; COUNT]) -> PointerGraph<'_> { + PointerGraph { + owners, + scratch: Region::new(0xffff_fc22_0000_0000, 0x8000).unwrap(), + pool_alias: 0x70_0000_0000, + parameter_alias: 0x70_0000_8000, + } + } + + #[test] + fn complete_plan_has_disjoint_page_owners_and_nine_log_threads() { + let owners = owners(0); + assert!(graph(&owners).addresses().is_ok()); + assert_eq!(allocation(28).unwrap().size, 0x1b0); + assert_eq!(allocation(29).unwrap().size, 0x28800); + assert_eq!(allocation(41).unwrap().size, 0x79800); + assert!(allocation(COUNT).is_err()); + } + + #[test] + fn runtime_references_relocate_and_keep_the_packed_power_pointer() { + for delta in [0, 0x1000_0000] { + let owners = owners(delta); + let mut output = [0xa5; RUNTIME_SIZE + 8]; + graph(&owners).runtime(&mut output).unwrap(); + let read = |at| u64::from_le_bytes(output[at..at + 8].try_into().unwrap()); + assert_eq!( + read(0x469), + allocation(PERFORMANCE_CONTROL).unwrap().address + delta + ); + assert_eq!(read(0x1a0), allocation(24).unwrap().address + delta); + assert_eq!(read(0x1b8), allocation(25).unwrap().address + delta); + assert_eq!(read(0x2d0), 0x70_0000_8000); + assert_eq!(&output[RUNTIME_SIZE..], &[0xa5; 8]); + } + } + + #[test] + fn every_short_runtime_rejects_before_mutation() { + let owners = owners(0); + for length in 0..RUNTIME_SIZE { + let mut output = [0xa5; RUNTIME_SIZE]; + assert_eq!( + graph(&owners).runtime(&mut output[..length]), + Err(Error::Size) + ); + assert_eq!(output, [0xa5; RUNTIME_SIZE]); + } + } + + #[test] + fn every_owner_is_checked_before_any_record_is_written() { + for index in 0..COUNT { + let mut owners = owners(0); + let item = allocation(index).unwrap(); + owners[index] = Region::new(item.address, item.size - 1).unwrap(); + let mut output = [0xa5; RUNTIME_SIZE]; + assert!(graph(&owners).runtime(&mut output).is_err()); + assert_eq!(output, [0xa5; RUNTIME_SIZE]); + } + let mut owners = owners(0); + owners[1] = owners[0]; + let mut output = [0xa5; RUNTIME_SIZE]; + assert!(graph(&owners).runtime(&mut output).is_err()); + assert_eq!(output, [0xa5; RUNTIME_SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_init_layout.rs b/drivers/gpu/drm/asahi/m3/m3_init_layout.rs index 92cc818e030cc..75f06abe6f411 100644 --- a/drivers/gpu/drm/asahi/m3/m3_init_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_init_layout.rs @@ -78,13 +78,38 @@ impl Root { (root::CONTROL, self.control.at(0, CONTROL_SIZE)?), (root::POWER, self.power.at(0, 2292)?), ]; - let bytes = out.get_mut(..ROOT_SIZE).ok_or(Error::Size)?; + Self::write_root(out, ROOT_SIZE, 0x0c08_e21e_8380_0490, &references) + } + + /// Encode the extended root without admitting a firmware image or publishing it. + pub(crate) fn encode_26(&self, out: &mut [u8], extra: Region) -> Result<(), Error> { + let references = [ + (root::REGION_A, self.region_a.at(0, 0x4000)?), + (root::RUNTIME, self.runtime.at(0, 0x4b8)?), + (root::GLOBALS, self.globals.at(0, 0xe48)?), + (root::NEW_A, self.new_a.at(0, 0x20)?), + (root::CONTROL, self.control.at(0, 0xeaf0)?), + (root::POWER, self.power.at(0, 0x670)?), + (0xc0, extra.at(0, 0x4000)?), + ]; + if references.iter().any(|(_, address)| address & 7 != 0) { + return Err(Error::Address); + } + Self::write_root(out, 0xc8, 0x0c89_c357_8392_04b8, &references) + } + + fn write_root( + out: &mut [u8], + size: usize, + version: u64, + references: &[(usize, u64)], + ) -> Result<(), Error> { + let bytes = out.get_mut(..size).ok_or(Error::Size)?; bytes.fill(0); - // Opaque qualified G15/V14_8_3 version tuple, not a semantic version. - for (i, value) in [0x490u16, 0x8380, 0xe21e, 0xc08].iter().enumerate() { - bytes[root::VERSION + i * 2..root::VERSION + i * 2 + 2].copy_from_slice(&value.to_le_bytes()); + pointer(bytes, root::VERSION, version); + for &(offset, value) in references { + pointer(bytes, offset, value); } - for (offset, value) in references { pointer(bytes, offset, value); } word(bytes, root::HOST_ALLOCATIONS, 1); bytes[root::PAGE_SIZE..root::PAGE_SIZE + 2].copy_from_slice(&0x4000u16.to_le_bytes()); bytes[root::PAGE_BITS] = 14; @@ -193,3 +218,107 @@ impl ControlRegion { Ok(()) } } + +#[cfg(test)] +mod root_format_tests { + use super::*; + + fn allocations() -> (Root, Region) { + let region = + |index: u64| Region::new(0xffff_fc01_0000_0000 + index * 0x10000, 0x10000).unwrap(); + ( + Root { + region_a: region(1), + runtime: region(2), + globals: region(3), + new_a: region(4), + control: region(5), + power: region(6), + }, + region(7), + ) + } + + #[test] + fn root_revision_changes_only_the_version_and_extension() { + let (root, extra) = allocations(); + let mut older = [0x55; 0xd0]; + let mut newer = older; + root.encode(&mut older).unwrap(); + root.encode_26(&mut newer, extra).unwrap(); + assert_eq!(&older[8..0xc0], &newer[8..0xc0]); + assert_eq!( + u64::from_le_bytes(newer[..8].try_into().unwrap()), + 0x0c89_c357_8392_04b8 + ); + assert_eq!( + u64::from_le_bytes(newer[0xc0..0xc8].try_into().unwrap()), + 0xffff_fc01_0007_0000 + ); + assert_eq!(&older[0xc0..], &[0x55; 16]); + assert_eq!(&newer[0xc8..], &[0x55; 8]); + for (offset, index) in [ + (8, 1), + (0x18, 2), + (0x20, 3), + (0xa8, 4), + (0xb0, 5), + (0xb8, 6), + ] { + assert_eq!( + u64::from_le_bytes(newer[offset..offset + 8].try_into().unwrap()), + 0xffff_fc01_0000_0000 + index * 0x10000 + ); + } + } + + #[test] + fn every_short_destination_is_unchanged() { + let (root, extra) = allocations(); + for length in 0..0xc8 { + let mut output = [0xa5; 0xc8]; + assert_eq!( + root.encode_26(&mut output[..length], extra), + Err(Error::Size) + ); + assert_eq!(output, [0xa5; 0xc8]); + } + } + + #[test] + fn every_reference_requires_its_own_complete_allocation() { + for (index, required) in [0x4000, 0x4b8, 0xe48, 0x20, 0xeaf0, 0x670, 0x4000] + .into_iter() + .enumerate() + { + for (size, aligned, valid) in [ + (required - 1, true, false), + (required, false, false), + (required, true, true), + ] { + let (mut root, mut extra) = allocations(); + let reference = match index { + 0 => &mut root.region_a, + 1 => &mut root.runtime, + 2 => &mut root.globals, + 3 => &mut root.new_a, + 4 => &mut root.control, + 5 => &mut root.power, + _ => &mut extra, + }; + *reference = + Region::new(0xffff_fc02_0000_0000 + u64::from(!aligned), size).unwrap(); + let mut output = [0xa5; 0xc8]; + let result = root.encode_26(&mut output, extra); + assert_eq!( + result.is_ok(), + valid, + "reference {index}, size {size}, aligned {aligned}" + ); + if !valid { + assert_eq!(output, [0xa5; 0xc8]); + } + } + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs b/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs new file mode 100644 index 0000000000000..e4008f239c4a0 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T8122 firmware MMIO descriptors. Mapping ownership stays with the M3 runtime. + +use super::m3_init_layout::{Error, Region}; + +pub(crate) const SLOTS: usize = 53; +pub(crate) const RECORD_BYTES: usize = 40; +pub(crate) const TABLE_OFFSET: usize = 0x640; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Area { + pub(crate) physical: u64, + pub(crate) bytes: usize, + relative_register: u64, + writable: bool, + split: bool, +} + +impl Area { + pub(crate) fn page_offset(self) -> usize { + (self.physical & 0x3fff) as usize + } + + pub(crate) fn mapped_bytes(self) -> usize { + (self.page_offset() + self.bytes + 0x3fff) & !0x3fff + } + + /// At most two physical banks; the second bank occupies the next virtual pages. + pub(crate) fn banks(self) -> [(u64, usize, usize); 2] { + if self.split { + [ + (self.physical, self.bytes / 2, 0), + (self.physical + 0x200_0000, self.bytes / 2, self.bytes / 2), + ] + } else { + [(self.physical & !0x3fff, self.mapped_bytes(), 0), (0, 0, 0)] + } + } +} + +/// Register resources obtained from the admitted T8122 hardware configuration. +/// Their physical addresses are supplied by the owner, not duplicated in a wire table. +#[derive(Clone, Copy)] +pub(crate) struct RegisterBases { + pub(crate) gpu: u64, + pub(crate) interrupt: u64, + pub(crate) memory: u64, + pub(crate) telemetry_write: u64, + pub(crate) telemetry_read: u64, + pub(crate) neural: u64, + pub(crate) metrology: u64, +} + +/// Select the wire slot and map its logical hardware block independently of its VA. +pub(crate) fn area(slot: usize, bases: RegisterBases) -> Result, Error> { + let (base, offset, bytes, tagged, writable, split) = match slot { + 0 => (bases.interrupt, 0, 0x4000, false, true, false), + 3 => (bases.memory, 0, 0x30000, false, true, true), + 9 => (bases.telemetry_write, 0, 0x1000, false, true, false), + 10 => (bases.telemetry_read, 0, 0x2000, false, false, false), + 12 => (bases.neural, 0, 0x4000, false, true, false), + 14 => (bases.metrology, 0, 0x8000, false, false, false), + 17 => (bases.gpu, 0, 0x20000, false, true, false), + 26 => (bases.gpu, 0xd04000, 0x8000, true, true, false), + 27 => (bases.gpu, 0xd0d000, 0x1000, true, true, false), + 28 => (bases.gpu, 0xd50000, 0x8000, true, true, false), + 29 => (bases.gpu, 0xd10000, 0x4000, true, true, false), + 31 => (bases.gpu, 0xd40000, 0x4000, true, true, false), + 32 => (bases.gpu, 0xd60000, 0x8000, true, true, false), + 35 => (bases.gpu, 0xe00000, 0x4000, true, true, false), + 39 => (bases.gpu, 0xe08000, 0x8000, false, true, false), + 40 => (bases.gpu, 0xe1c000, 0x4000, true, true, false), + 0..SLOTS => return Ok(None), + _ => return Err(Error::Bounds), + }; + let physical = base.checked_add(offset).ok_or(Error::Address)?; + if physical == 0 || physical.checked_add(bytes as u64).ok_or(Error::Address)? > 1 << 42 { + return Err(Error::Address); + } + Ok(Some(Area { + physical, + bytes, + relative_register: if tagged { offset } else { 0 }, + writable, + split, + })) +} + +fn empty_mode(slot: usize) -> u64 { + if matches!(slot, 2 | 5..=8 | 30 | 33 | 34 | 37 | 38 | 41..=43 | 46..=49 | 52) { + 2 + } else { + 0 + } +} + +/// Encode a complete record after checking the owned mapping; errors preserve all output. +pub(crate) fn encode( + slot: usize, + bases: RegisterBases, + mapping: Option, + output: &mut [u8], +) -> Result<(), Error> { + let descriptor = area(slot, bases)?; + let (physical, va, bytes, requested, register, mode) = match (descriptor, mapping) { + (None, None) => (0, 0, 0, 0, 0, empty_mode(slot)), + (Some(item), Some(mapping)) => { + let page = mapping.at(0, item.mapped_bytes())?; + if page & 0x3fff != 0 { + return Err(Error::Address); + } + let va = page + .checked_add(item.page_offset() as u64) + .ok_or(Error::Address)?; + ( + item.physical, + va, + item.bytes as u32, + if item.split { + (item.bytes / 2) as u32 + } else { + item.bytes as u32 + }, + item.relative_register, + if item.writable { 2 } else { 0 }, + ) + } + _ => return Err(Error::Address), + }; + let bytes_out = output.get_mut(..RECORD_BYTES).ok_or(Error::Size)?; + bytes_out.fill(0); + bytes_out[0..8].copy_from_slice(&physical.to_le_bytes()); + bytes_out[8..16].copy_from_slice(&va.to_le_bytes()); + bytes_out[16..20].copy_from_slice(&bytes.to_le_bytes()); + bytes_out[20..24].copy_from_slice(&requested.to_le_bytes()); + bytes_out[24..32].copy_from_slice(®ister.to_le_bytes()); + bytes_out[32..40].copy_from_slice(&mode.to_le_bytes()); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Register identities already present in the public T8122 configuration; + // memory-controller bank geometry is shared with the public 25G83 encoder. + const BASES: RegisterBases = RegisterBases { + gpu: 0x2_9000_0000, + interrupt: 0x2_d101_4000, + memory: 0x2_201c_4000, + telemetry_write: 0x2_d03d_0000, + telemetry_read: 0x2_d03c_0000, + neural: 0x3_1145_c000, + metrology: 0x2_d028_0000, + }; + + #[test] + fn j613_slot28_bank_and_wire_tag_agree() { + let bank = area(28, BASES).unwrap().unwrap(); + assert_eq!(bank.physical, BASES.gpu + 0xd50000); + assert_eq!(bank.bytes, 0x8000); + let mapped = Region::new(0xffff_fc23_01c0_0000, 0x8000).unwrap(); + let mut bytes = [0; RECORD_BYTES]; + encode(28, BASES, Some(mapped), &mut bytes).unwrap(); + assert_eq!(&bytes[0..8], &(BASES.gpu + 0xd50000).to_le_bytes()); + assert_eq!(&bytes[24..32], &0xd50000u64.to_le_bytes()); + assert_ne!(bank.physical, BASES.gpu + 0xd58000); + } + + #[test] + fn descriptor_inventory_ends_before_the_retention_pointer() { + assert_eq!(TABLE_OFFSET + SLOTS * RECORD_BYTES, 0xe88); + assert_eq!( + (0..SLOTS) + .filter(|slot| area(*slot, BASES).unwrap().is_some()) + .count(), + 16 + ); + assert!(area(SLOTS, BASES).is_err()); + } + + #[test] + fn discontiguous_memory_controller_banks_have_distinct_physical_owners() { + let item = area(3, BASES).unwrap().unwrap(); + assert_eq!( + item.banks(), + [ + (0x2_201c_4000, 0x18000, 0), + (0x2_221c_4000, 0x18000, 0x18000) + ] + ); + let mapping = Region::new(0xffff_fc23_0000_0000, 0x30000).unwrap(); + let mut bytes = [0xa5; RECORD_BYTES]; + encode(3, BASES, Some(mapping), &mut bytes).unwrap(); + assert_eq!( + u32::from_le_bytes(bytes[16..20].try_into().unwrap()), + 0x30000 + ); + assert_eq!( + u32::from_le_bytes(bytes[20..24].try_into().unwrap()), + 0x18000 + ); + } + + #[test] + fn subpage_mapping_keeps_the_register_offset() { + let item = area(27, BASES).unwrap().unwrap(); + assert_eq!(item.page_offset(), 0x1000); + assert_eq!(item.banks()[0], (0x2_90d0_c000, 0x4000, 0)); + let mut output = [0xa5; RECORD_BYTES + 8]; + encode( + 27, + BASES, + Some(Region::new(0xffff_fc23_0000_0000, 0x4000).unwrap()), + &mut output, + ) + .unwrap(); + assert_eq!( + u64::from_le_bytes(output[8..16].try_into().unwrap()), + 0xffff_fc23_0000_1000 + ); + assert_eq!(&output[RECORD_BYTES..], &[0xa5; 8]); + } + + #[test] + fn empty_slots_keep_modes_without_fabricating_mappings() { + for slot in 0..SLOTS { + if area(slot, BASES).unwrap().is_some() { + continue; + } + let mut output = [0xa5; RECORD_BYTES]; + encode(slot, BASES, None, &mut output).unwrap(); + assert_eq!(&output[..32], &[0; 32]); + assert_eq!( + u64::from_le_bytes(output[32..40].try_into().unwrap()), + empty_mode(slot) + ); + } + } + + #[test] + fn invalid_mapping_or_output_never_partially_changes_a_descriptor() { + let good = Region::new(0xffff_fc23_0000_0000, 0x4000).unwrap(); + for (slot, mapping) in [ + (SLOTS, None), + (0, None), + (1, Some(good)), + (0, Some(Region::new(0xffff_fc23_0000_0001, 0x4000).unwrap())), + (0, Some(Region::new(0xffff_fc23_0000_0000, 0x3fff).unwrap())), + ] { + let mut output = [0xa5; RECORD_BYTES]; + assert!(encode(slot, BASES, mapping, &mut output).is_err()); + assert_eq!(output, [0xa5; RECORD_BYTES]); + } + for length in 0..RECORD_BYTES { + let mut output = [0xa5; RECORD_BYTES]; + assert_eq!( + encode(0, BASES, Some(good), &mut output[..length]), + Err(Error::Size) + ); + assert_eq!(output, [0xa5; RECORD_BYTES]); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_profile.rs b/drivers/gpu/drm/asahi/m3/m3_profile.rs new file mode 100644 index 0000000000000..c70a88029072b --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_profile.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware-interface properties, independent of the physical G15 GPU record format. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Profile { + Legacy, + Extended, +} + +pub(crate) const PARAMS_REQUIRED: u64 = 600; + +impl Profile { + pub(crate) const fn firmware_compat(self) -> [u32; 3] { + match self { + Self::Legacy => [14, 8, 3], + Self::Extended => [26, 6, 2], + } + } + + pub(crate) const fn hal(self) -> u32 { + match self { + Self::Legacy => 0, + Self::Extended => 200, + } + } + + pub(crate) const fn user_end(self) -> u64 { + match self { + Self::Legacy => 0x2ff_ffff_8000, + Self::Extended => (1 << 42) - 2 * 0x4000, + } + } + + pub(crate) const fn vm_acknowledged(self, params_bytes: u64, flags: u32) -> bool { + // Both profiles consume G15 records; the existing VM padding stays zero. + params_bytes >= PARAMS_REQUIRED && flags == 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn profile_acknowledgement_is_exact_and_file_query_covers_hal() { + for profile in [Profile::Legacy, Profile::Extended] { + let expected = 0; + for size in [0, 599, 600, 616, u64::MAX] { + for flags in [0, 1, 2, 3, u32::MAX] { + assert_eq!( + profile.vm_acknowledged(size, flags), + size >= 600 && flags == expected + ); + } + } + } + } + + #[test] + fn firmware_interface_version_does_not_change_physical_identity() { + assert_eq!(Profile::Legacy.hal(), 0); + assert_eq!(Profile::Extended.hal(), 200); + assert_eq!(Profile::Legacy.firmware_compat(), [14, 8, 3]); + assert_eq!(Profile::Extended.firmware_compat(), [26, 6, 2]); + assert_eq!(Profile::Extended.user_end(), 0x3ff_ffff_8000); + assert_eq!(Profile::Legacy.user_end(), 0x2ff_ffff_8000); + } +} diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index 1d10657c02f66..7073ecf294d63 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -198,3 +198,7 @@ mod t8122_admission; pub(crate) mod m3_soc; mod t8122_hwdata; + +mod m3_init_26; +mod m3_profile; +mod m3_mmio_26; diff --git a/drivers/gpu/drm/asahi/m3/t8122_admission.rs b/drivers/gpu/drm/asahi/m3/t8122_admission.rs index 12bea894c373e..fdc41024899f1 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_admission.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_admission.rs @@ -2,6 +2,8 @@ //! Read-only T8122 identity admission. Firmware startup remains disabled. +use super::m3_profile::Profile; + #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) struct Identity { pub(crate) active_cores: u32, @@ -16,12 +18,23 @@ pub(crate) fn validate_identity( result: u32, id_version: u32, masks: [u32; 3], +) -> Option { + validate_identity_for(Profile::Legacy, firmware_compat, schema, result, id_version, masks) +} + +pub(crate) fn validate_identity_for( + profile: Profile, + firmware_compat: &[u32], + schema: u32, + result: u32, + id_version: u32, + masks: [u32; 3], ) -> Option { let family = id_version >> 24; let variant = (id_version >> 16) & 0xff; let revision = (id_version >> 8) & 0xff; let active_cores = masks[0].count_ones(); - if firmware_compat != [14, 8, 3] + if firmware_compat != profile.firmware_compat() || schema != 1 || result != 1 || family != 7 @@ -94,6 +107,14 @@ pub(crate) fn opp_table_shape(opps: impl IntoIterator) -> Opt #[cfg(not(test))] pub(crate) fn read_identity( pdev: &kernel::platform::Device>, +) -> kernel::error::Result { + read_identity_for(pdev, Profile::Legacy) +} + +#[cfg(not(test))] +pub(crate) fn read_identity_for( + pdev: &kernel::platform::Device>, + profile: Profile, ) -> kernel::error::Result { use kernel::{bindings, c_str, prelude::*}; @@ -146,7 +167,8 @@ pub(crate) fn read_identity( observation.get_u32(c_str!("core-mask-1"))?, observation.get_u32(c_str!("core-mask-2"))?, ]; - validate_identity(firmware_compat.as_slice(), schema, result, id_version, masks).ok_or(ENODEV) + validate_identity_for(profile, firmware_compat.as_slice(), schema, result, id_version, masks) + .ok_or(ENODEV) } #[cfg(test)] @@ -164,6 +186,29 @@ mod tests { } } + #[test] + fn extended_identity_requires_its_actual_version_and_the_same_g15g_bounds() { + for mask in [0xff, 0x3ff] { + let good = || validate_identity_for(Profile::Extended, &[26, 6, 2], 1, 1, G15G_C0, [mask, 0, 0]); + assert_eq!(good().unwrap().active_cores, mask.count_ones()); + assert!(validate_identity(&[26, 6, 2], 1, 1, G15G_C0, [mask, 0, 0]).is_none()); + assert!(validate_identity_for(Profile::Extended, &[14, 8, 3], 1, 1, G15G_C0, [mask, 0, 0]).is_none()); + } + for (schema, result, id, masks) in [ + (0, 1, G15G_C0, [0xff, 0, 0]), + (1, 0, G15G_C0, [0xff, 0, 0]), + (1, 1, G15G_C0 ^ (1 << 24), [0xff, 0, 0]), + (1, 1, G15G_C0 ^ (1 << 16), [0xff, 0, 0]), + (1, 1, G15G_C0 ^ (1 << 8), [0xff, 0, 0]), + (1, 1, G15G_C0, [0x1ff, 0, 0]), + (1, 1, G15G_C0, [0x7ff, 0, 0]), + (1, 1, G15G_C0, [0xff, 1, 0]), + (1, 1, G15G_C0, [0xff, 0, 1]), + ] { + assert!(validate_identity_for(Profile::Extended, &[26, 6, 2], schema, result, id, masks).is_none()); + } + } + /// The booted J613 ladder (`perf-states`, frequency then millivolts), in ADT order: fourteen /// states, eight voltages above the idle one, 1338 MHz at the top. const J613_PERF_STATES: [(u64, u32); 14] = [ diff --git a/tools/asahi/m3-firmware-tests.rs b/tools/asahi/m3-firmware-tests.rs new file mode 100644 index 0000000000000..90539441d623e --- /dev/null +++ b/tools/asahi/m3-firmware-tests.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Host tests of production firmware identity and normalization. +#![allow(dead_code)] +#[path = "../../drivers/gpu/drm/asahi/m3/m3_profile.rs"] +pub(crate) mod m3_profile; +#[path = "../../drivers/gpu/drm/asahi/m3/agx_resources.rs"] +pub(crate) mod resources; +mod m3 { + pub(crate) use crate::firmware as m3_firmware; + pub(crate) use crate::m3_profile; + pub(crate) use crate::resources as m3_resources; +} +#[path = "../../drivers/gpu/drm/asahi/m3/m3_firmware.rs"] +pub(crate) mod firmware; diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index fd4dc90489e81..f1768a242147b 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -15,3 +15,12 @@ mod m3_init_storage; mod m3_thermal_policy; #[path = "../../drivers/gpu/drm/asahi/m3/t8122_hwdata.rs"] mod t8122_hwdata; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_init_26.rs"] +mod m3_init_26; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_profile.rs"] +mod m3_profile; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_mmio_26.rs"] +mod m3_mmio_26; diff --git a/tools/asahi/m3-uapi-fixture.c b/tools/asahi/m3-uapi-fixture.c new file mode 100644 index 0000000000000..9ee0994096009 --- /dev/null +++ b/tools/asahi/m3-uapi-fixture.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include + +#ifndef ASAHI_HEADER +#define ASAHI_HEADER "drm/asahi_drm.h" +#endif +#include ASAHI_HEADER + +static_assert(sizeof(struct drm_asahi_cmd_header) == 8, "command header"); +static_assert(sizeof(struct drm_asahi_attachment) == 24, "attachment"); +static_assert(sizeof(struct drm_asahi_timestamps) == 16, "timestamps"); +static_assert(offsetof(struct drm_asahi_cmd_compute, ts_end_copies) == 64, + "compute prefix"); +static_assert(offsetof(struct drm_asahi_cmd_render, ts_frag_end_copies) == 256, + "render prefix"); + +static int emit(unsigned int type, const void *payload, size_t size) +{ + struct drm_asahi_cmd_header header = { + .cmd_type = type, + .size = size, + .vdm_barrier = DRM_ASAHI_BARRIER_NONE, + .cdm_barrier = DRM_ASAHI_BARRIER_NONE, + }; + + return fwrite(&header, sizeof(header), 1, stdout) != 1 || + fwrite(payload, size, 1, stdout) != 1; +} + +/* Named C offsets are exported independently of the Rust decoder. */ +static void layout(void) +{ + printf("profile.params_required %zu 0\n", + offsetof(struct drm_asahi_params_global, gpu_hal_generation) + sizeof(__u32)); + printf("profile.vm_create_size %zu 0\n", sizeof(struct drm_asahi_vm_create)); + printf("profile.hal200 %u 0\n", (unsigned int)DRM_ASAHI_GPU_HAL_200); + printf("profile.features %u 0\n", + DRM_ASAHI_FEATURE_VM_STATUS | DRM_ASAHI_FEATURE_SCHEDULED_QUEUES); + printf("header %zu 0\n", sizeof(struct drm_asahi_cmd_header)); + printf("attachment %zu 0\n", sizeof(struct drm_asahi_attachment)); + printf("timestamps %zu 0\n", sizeof(struct drm_asahi_timestamps)); + printf("cmd_render.flags %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, flags), + sizeof(((struct drm_asahi_cmd_render *)0)->flags)); + printf("cmd_render.isp_zls_pixels %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_zls_pixels), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_zls_pixels)); + printf("cmd_render.vdm_ctrl_stream_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, vdm_ctrl_stream_base), + sizeof(((struct drm_asahi_cmd_render *)0)->vdm_ctrl_stream_base)); + printf("cmd_render.vertex_helper.binary %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, vertex_helper.binary), + sizeof(((struct drm_asahi_cmd_render *)0)->vertex_helper.binary)); + printf("cmd_render.vertex_helper.cfg %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, vertex_helper.cfg), + sizeof(((struct drm_asahi_cmd_render *)0)->vertex_helper.cfg)); + printf("cmd_render.vertex_helper.data %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, vertex_helper.data), + sizeof(((struct drm_asahi_cmd_render *)0)->vertex_helper.data)); + printf("cmd_render.fragment_helper.binary %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, fragment_helper.binary), + sizeof(((struct drm_asahi_cmd_render *)0)->fragment_helper.binary)); + printf("cmd_render.fragment_helper.cfg %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, fragment_helper.cfg), + sizeof(((struct drm_asahi_cmd_render *)0)->fragment_helper.cfg)); + printf("cmd_render.fragment_helper.data %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, fragment_helper.data), + sizeof(((struct drm_asahi_cmd_render *)0)->fragment_helper.data)); + printf("cmd_render.isp_scissor_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_scissor_base), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_scissor_base)); + printf("cmd_render.isp_dbias_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_dbias_base), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_dbias_base)); + printf("cmd_render.isp_oclqry_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_oclqry_base), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_oclqry_base)); + printf("cmd_render.depth.base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, depth.base), + sizeof(((struct drm_asahi_cmd_render *)0)->depth.base)); + printf("cmd_render.depth.comp_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, depth.comp_base), + sizeof(((struct drm_asahi_cmd_render *)0)->depth.comp_base)); + printf("cmd_render.depth.stride %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, depth.stride), + sizeof(((struct drm_asahi_cmd_render *)0)->depth.stride)); + printf("cmd_render.depth.comp_stride %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, depth.comp_stride), + sizeof(((struct drm_asahi_cmd_render *)0)->depth.comp_stride)); + printf("cmd_render.stencil.base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, stencil.base), + sizeof(((struct drm_asahi_cmd_render *)0)->stencil.base)); + printf("cmd_render.stencil.comp_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, stencil.comp_base), + sizeof(((struct drm_asahi_cmd_render *)0)->stencil.comp_base)); + printf("cmd_render.stencil.stride %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, stencil.stride), + sizeof(((struct drm_asahi_cmd_render *)0)->stencil.stride)); + printf("cmd_render.stencil.comp_stride %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, stencil.comp_stride), + sizeof(((struct drm_asahi_cmd_render *)0)->stencil.comp_stride)); + printf("cmd_render.zls_ctrl %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, zls_ctrl), + sizeof(((struct drm_asahi_cmd_render *)0)->zls_ctrl)); + printf("cmd_render.ppp_multisamplectl %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ppp_multisamplectl), + sizeof(((struct drm_asahi_cmd_render *)0)->ppp_multisamplectl)); + printf("cmd_render.sampler_heap %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, sampler_heap), + sizeof(((struct drm_asahi_cmd_render *)0)->sampler_heap)); + printf("cmd_render.ppp_ctrl %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ppp_ctrl), + sizeof(((struct drm_asahi_cmd_render *)0)->ppp_ctrl)); + printf("cmd_render.width_px %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, width_px), + sizeof(((struct drm_asahi_cmd_render *)0)->width_px)); + printf("cmd_render.height_px %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, height_px), + sizeof(((struct drm_asahi_cmd_render *)0)->height_px)); + printf("cmd_render.layers %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, layers), + sizeof(((struct drm_asahi_cmd_render *)0)->layers)); + printf("cmd_render.sampler_count %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, sampler_count), + sizeof(((struct drm_asahi_cmd_render *)0)->sampler_count)); + printf("cmd_render.utile_width_px %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, utile_width_px), + sizeof(((struct drm_asahi_cmd_render *)0)->utile_width_px)); + printf("cmd_render.utile_height_px %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, utile_height_px), + sizeof(((struct drm_asahi_cmd_render *)0)->utile_height_px)); + printf("cmd_render.samples %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, samples), + sizeof(((struct drm_asahi_cmd_render *)0)->samples)); + printf("cmd_render.sample_size_B %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, sample_size_B), + sizeof(((struct drm_asahi_cmd_render *)0)->sample_size_B)); + printf("cmd_render.isp_merge_upper_x %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_merge_upper_x), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_merge_upper_x)); + printf("cmd_render.isp_merge_upper_y %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_merge_upper_y), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_merge_upper_y)); + printf("cmd_render.bg.usc %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg.usc), + sizeof(((struct drm_asahi_cmd_render *)0)->bg.usc)); + printf("cmd_render.bg.rsrc_spec %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg.rsrc_spec), + sizeof(((struct drm_asahi_cmd_render *)0)->bg.rsrc_spec)); + printf("cmd_render.bg_rsrc_spec_hi %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg_rsrc_spec_hi), + sizeof(((struct drm_asahi_cmd_render *)0)->bg_rsrc_spec_hi)); + printf("cmd_render.eot.usc %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, eot.usc), + sizeof(((struct drm_asahi_cmd_render *)0)->eot.usc)); + printf("cmd_render.eot.rsrc_spec %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, eot.rsrc_spec), + sizeof(((struct drm_asahi_cmd_render *)0)->eot.rsrc_spec)); + printf("cmd_render.bg_eot_rsrc_spec_hi %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg_eot_rsrc_spec_hi), + sizeof(((struct drm_asahi_cmd_render *)0)->bg_eot_rsrc_spec_hi)); + printf("cmd_render.partial_bg.usc %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, partial_bg.usc), + sizeof(((struct drm_asahi_cmd_render *)0)->partial_bg.usc)); + printf("cmd_render.partial_bg.rsrc_spec %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, partial_bg.rsrc_spec), + sizeof(((struct drm_asahi_cmd_render *)0)->partial_bg.rsrc_spec)); + printf("cmd_render.bg_partial_rsrc_spec_hi %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg_partial_rsrc_spec_hi), + sizeof(((struct drm_asahi_cmd_render *)0)->bg_partial_rsrc_spec_hi)); + printf("cmd_render.partial_eot.usc %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, partial_eot.usc), + sizeof(((struct drm_asahi_cmd_render *)0)->partial_eot.usc)); + printf("cmd_render.partial_eot.rsrc_spec %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, partial_eot.rsrc_spec), + sizeof(((struct drm_asahi_cmd_render *)0)->partial_eot.rsrc_spec)); + printf("cmd_render.bg_eot_partial_rsrc_spec_hi %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, bg_eot_partial_rsrc_spec_hi), + sizeof(((struct drm_asahi_cmd_render *)0)->bg_eot_partial_rsrc_spec_hi)); + printf("cmd_render.isp_bgobjdepth %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_bgobjdepth), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_bgobjdepth)); + printf("cmd_render.isp_bgobjvals %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, isp_bgobjvals), + sizeof(((struct drm_asahi_cmd_render *)0)->isp_bgobjvals)); + printf("cmd_render.ts_vtx.start.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_vtx.start.handle), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.start.handle)); + printf("cmd_render.ts_vtx.start.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_vtx.start.offset), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.start.offset)); + printf("cmd_render.ts_vtx.end.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_vtx.end.handle), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.end.handle)); + printf("cmd_render.ts_vtx.end.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_vtx.end.offset), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_vtx.end.offset)); + printf("cmd_render.ts_frag.start.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_frag.start.handle), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.start.handle)); + printf("cmd_render.ts_frag.start.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_frag.start.offset), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.start.offset)); + printf("cmd_render.ts_frag.end.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_frag.end.handle), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.end.handle)); + printf("cmd_render.ts_frag.end.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_frag.end.offset), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag.end.offset)); + printf("cmd_compute.flags %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, flags), + sizeof(((struct drm_asahi_cmd_compute *)0)->flags)); + printf("cmd_compute.sampler_count %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, sampler_count), + sizeof(((struct drm_asahi_cmd_compute *)0)->sampler_count)); + printf("cmd_compute.cdm_ctrl_stream_base %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, cdm_ctrl_stream_base), + sizeof(((struct drm_asahi_cmd_compute *)0)->cdm_ctrl_stream_base)); + printf("cmd_compute.cdm_ctrl_stream_end %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, cdm_ctrl_stream_end), + sizeof(((struct drm_asahi_cmd_compute *)0)->cdm_ctrl_stream_end)); + printf("cmd_compute.sampler_heap %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, sampler_heap), + sizeof(((struct drm_asahi_cmd_compute *)0)->sampler_heap)); + printf("cmd_compute.helper.binary %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, helper.binary), + sizeof(((struct drm_asahi_cmd_compute *)0)->helper.binary)); + printf("cmd_compute.helper.cfg %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, helper.cfg), + sizeof(((struct drm_asahi_cmd_compute *)0)->helper.cfg)); + printf("cmd_compute.helper.data %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, helper.data), + sizeof(((struct drm_asahi_cmd_compute *)0)->helper.data)); + printf("cmd_compute.ts.start.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, ts.start.handle), + sizeof(((struct drm_asahi_cmd_compute *)0)->ts.start.handle)); + printf("cmd_compute.ts.start.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, ts.start.offset), + sizeof(((struct drm_asahi_cmd_compute *)0)->ts.start.offset)); + printf("cmd_compute.ts.end.handle %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, ts.end.handle), + sizeof(((struct drm_asahi_cmd_compute *)0)->ts.end.handle)); + printf("cmd_compute.ts.end.offset %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, ts.end.offset), + sizeof(((struct drm_asahi_cmd_compute *)0)->ts.end.offset)); + printf("attachment.pointer %zu %zu\n", + offsetof(struct drm_asahi_attachment, pointer), + sizeof(((struct drm_asahi_attachment *)0)->pointer)); + printf("attachment.size %zu %zu\n", + offsetof(struct drm_asahi_attachment, size), + sizeof(((struct drm_asahi_attachment *)0)->size)); + printf("cmd_render.ts_frag_end_copies %zu %zu\n", + offsetof(struct drm_asahi_cmd_render, ts_frag_end_copies), + sizeof(((struct drm_asahi_cmd_render *)0)->ts_frag_end_copies)); + printf("cmd_compute.ts_end_copies %zu %zu\n", + offsetof(struct drm_asahi_cmd_compute, ts_end_copies), + sizeof(((struct drm_asahi_cmd_compute *)0)->ts_end_copies)); +} + +int main(int argc, char **argv) +{ + struct drm_asahi_attachment attachment = {0}; + + attachment.pointer = 0xcdb09376593c1f02ULL; + attachment.size = 0xceb194775a3d2003ULL; + + struct drm_asahi_cmd_render render = {0}; + + render.flags = 0x5b3e2104; + render.isp_zls_pixels = 0x5c3f2205; + render.vdm_ctrl_stream_base = 0xd1b4977a5d402306ULL; + render.vertex_helper.binary = 0x5e412407; + render.vertex_helper.cfg = 0x5f422508; + render.vertex_helper.data = 0xd4b79a7d60432609ULL; + render.fragment_helper.binary = 0x6144270a; + render.fragment_helper.cfg = 0x6245280b; + render.fragment_helper.data = 0xd7ba9d806346290cULL; + render.isp_scissor_base = 0xd8bb9e8164472a0dULL; + render.isp_dbias_base = 0xd9bc9f8265482b0eULL; + render.isp_oclqry_base = 0xdabda08366492c0fULL; + render.depth.base = 0xdbbea184674a2d10ULL; + render.depth.comp_base = 0xdcbfa285684b2e11ULL; + render.depth.stride = 0x694c2f12; + render.depth.comp_stride = 0x6a4d3013; + render.stencil.base = 0xdfc2a5886b4e3114ULL; + render.stencil.comp_base = 0xe0c3a6896c4f3215ULL; + render.stencil.stride = 0x6d503316; + render.stencil.comp_stride = 0x6e513417; + render.zls_ctrl = 0xe3c6a98c6f523518ULL; + render.ppp_multisamplectl = 0xe4c7aa8d70533619ULL; + render.sampler_heap = 0xe5c8ab8e7154371aULL; + render.ppp_ctrl = 0x7255381b; + render.width_px = 0x391c; + render.height_px = 0x3a1d; + render.layers = 0x3b1e; + render.sampler_count = 0x3c1f; + render.utile_width_px = 0x20; + render.utile_height_px = 0x21; + render.samples = 0x22; + render.sample_size_B = 0x23; + render.isp_merge_upper_x = 0x7b5e4124; + render.isp_merge_upper_y = 0x7c5f4225; + render.bg.usc = 0x7d604326; + render.bg.rsrc_spec = 0x7e614427; + render.bg_rsrc_spec_hi = 0x7f624528; + render.eot.usc = 0x80634629; + render.eot.rsrc_spec = 0x8164472a; + render.bg_eot_rsrc_spec_hi = 0x8265482b; + render.partial_bg.usc = 0x8366492c; + render.partial_bg.rsrc_spec = 0x84674a2d; + render.bg_partial_rsrc_spec_hi = 0x85684b2e; + render.partial_eot.usc = 0x86694c2f; + render.partial_eot.rsrc_spec = 0x876a4d30; + render.bg_eot_partial_rsrc_spec_hi = 0x886b4e31; + render.isp_bgobjdepth = 0x896c4f32; + render.isp_bgobjvals = 0x8a6d5033; + render.ts_vtx.start.handle = 0x8b6e5134; + render.ts_vtx.start.offset = 0x8c6f5235; + render.ts_vtx.end.handle = 0x8d705336; + render.ts_vtx.end.offset = 0x8e715437; + render.ts_frag.start.handle = 0x8f725538; + render.ts_frag.start.offset = 0x90735639; + render.ts_frag.end.handle = 0x9174573a; + render.ts_frag.end.offset = 0x9275583b; + + struct drm_asahi_cmd_compute compute = {0}; + + compute.flags = 0x9376593c; + compute.sampler_count = 0x94775a3d; + compute.cdm_ctrl_stream_base = 0xbeccfb295785b3eULL; + compute.cdm_ctrl_stream_end = 0xcedd0b396795c3fULL; + compute.sampler_heap = 0xdeed1b4977a5d40ULL; + compute.helper.binary = 0x987b5e41; + compute.helper.cfg = 0x997c5f42; + compute.helper.data = 0x10f1d4b79a7d6043ULL; + compute.ts.start.handle = 0x9b7e6144; + compute.ts.start.offset = 0x9c7f6245; + compute.ts.end.handle = 0x9d806346; + compute.ts.end.offset = 0x9e816447; + + size_t render_size = + offsetof(struct drm_asahi_cmd_render, ts_frag_end_copies); + size_t compute_size = + offsetof(struct drm_asahi_cmd_compute, ts_end_copies); + + if (argc == 2 && !strcmp(argv[1], "--layout")) { + layout(); + return 0; + } else if (argc == 2 && !strcmp(argv[1], "--full")) { + render_size = sizeof(render); + compute_size = sizeof(compute); + } else if (argc != 1) { + return 1; + } + + return emit(DRM_ASAHI_SET_VERTEX_ATTACHMENTS, &attachment, + sizeof(attachment)) || + emit(DRM_ASAHI_CMD_RENDER, &render, render_size) || + emit(DRM_ASAHI_CMD_COMPUTE, &compute, compute_size); +} diff --git a/tools/asahi/m3-uapi-tests.rs b/tools/asahi/m3-uapi-tests.rs new file mode 100644 index 0000000000000..4dc6012f5213e --- /dev/null +++ b/tools/asahi/m3-uapi-tests.rs @@ -0,0 +1,832 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Offline tests of the production parser. All addresses are synthetic. +#![allow(dead_code)] + +#[path = "../../drivers/gpu/drm/asahi/m3/t8122_admission.rs"] +mod admission; +#[path = "../../drivers/gpu/drm/asahi/m3/m3_profile.rs"] +mod m3_profile; +#[path = "../../drivers/gpu/drm/asahi/m3/agx_uapi.rs"] +mod uapi; +use uapi::*; + +fn frame(kind: u16, payload: &[u8]) -> Vec { + let mut bytes = Vec::new(); + for word in [ + kind, + payload.len() as u16, + UAPI_BARRIER_NONE, + UAPI_BARRIER_NONE, + ] { + bytes.extend_from_slice(&word.to_le_bytes()); + } + bytes.extend_from_slice(payload); + bytes +} + +fn parse(bytes: &[u8]) -> Result { + let mut parser = UapiCommandParser::new(bytes); + while parser.next_hardware()?.is_some() {} + parser.finish() +} + +// Each offset below is checked against offsetof of the corresponding C member. +fn layout() -> &'static std::collections::BTreeMap { + static LAYOUT: std::sync::OnceLock> = + std::sync::OnceLock::new(); + LAYOUT.get_or_init(|| { + let path = std::env::var("ASAHI_UAPI_LAYOUT").expect("C layout path"); + std::fs::read_to_string(path) + .unwrap() + .lines() + .map(|line| { + let words: Vec<_> = line.split_whitespace().collect(); + ( + words[0].to_string(), + (words[1].parse().unwrap(), words[2].parse().unwrap()), + ) + }) + .collect() + }) +} + +fn field(raw: &[u8], name: &str, offset: usize, width: usize, actual: u64) { + assert_eq!(layout()[name], (offset, width), "C offset/width for {name}"); + let mut expected = [0u8; 8]; + for (i, byte) in expected[..width].iter_mut().enumerate() { + *byte = raw.get(offset + i).copied().unwrap_or(0); + } + assert_eq!(actual, u64::from_le_bytes(expected), "decoded {name}"); +} + +fn check_render(p: UapiRenderCommand, raw: &[u8]) { + field(raw, "cmd_render.flags", 0, 4, p.flags as u64); + field( + raw, + "cmd_render.isp_zls_pixels", + 4, + 4, + p.depth_dimensions as u64, + ); + field( + raw, + "cmd_render.vdm_ctrl_stream_base", + 8, + 8, + p.vdm_base as u64, + ); + field( + raw, + "cmd_render.vertex_helper.binary", + 16, + 4, + p.vertex_helper.binary as u64, + ); + field( + raw, + "cmd_render.vertex_helper.cfg", + 20, + 4, + p.vertex_helper.config as u64, + ); + field( + raw, + "cmd_render.vertex_helper.data", + 24, + 8, + p.vertex_helper.data as u64, + ); + field( + raw, + "cmd_render.fragment_helper.binary", + 32, + 4, + p.fragment_helper.binary as u64, + ); + field( + raw, + "cmd_render.fragment_helper.cfg", + 36, + 4, + p.fragment_helper.config as u64, + ); + field( + raw, + "cmd_render.fragment_helper.data", + 40, + 8, + p.fragment_helper.data as u64, + ); + field( + raw, + "cmd_render.isp_scissor_base", + 48, + 8, + p.scissor_base as u64, + ); + field( + raw, + "cmd_render.isp_dbias_base", + 56, + 8, + p.depth_bias_base as u64, + ); + field( + raw, + "cmd_render.isp_oclqry_base", + 64, + 8, + p.occlusion_query_base as u64, + ); + field(raw, "cmd_render.depth.base", 72, 8, p.depth.base as u64); + field( + raw, + "cmd_render.depth.comp_base", + 80, + 8, + p.depth.compression_base as u64, + ); + field(raw, "cmd_render.depth.stride", 88, 4, p.depth.stride as u64); + field( + raw, + "cmd_render.depth.comp_stride", + 92, + 4, + p.depth.compression_stride as u64, + ); + field(raw, "cmd_render.stencil.base", 96, 8, p.stencil.base as u64); + field( + raw, + "cmd_render.stencil.comp_base", + 104, + 8, + p.stencil.compression_base as u64, + ); + field( + raw, + "cmd_render.stencil.stride", + 112, + 4, + p.stencil.stride as u64, + ); + field( + raw, + "cmd_render.stencil.comp_stride", + 116, + 4, + p.stencil.compression_stride as u64, + ); + field(raw, "cmd_render.zls_ctrl", 120, 8, p.zls_control as u64); + field( + raw, + "cmd_render.ppp_multisamplectl", + 128, + 8, + p.multisample_control as u64, + ); + field( + raw, + "cmd_render.sampler_heap", + 136, + 8, + p.sampler_heap as u64, + ); + field(raw, "cmd_render.ppp_ctrl", 144, 4, p.ppp_control as u64); + field(raw, "cmd_render.width_px", 148, 2, p.width as u64); + field(raw, "cmd_render.height_px", 150, 2, p.height as u64); + field(raw, "cmd_render.layers", 152, 2, p.layers as u64); + field( + raw, + "cmd_render.sampler_count", + 154, + 2, + p.sampler_count as u64, + ); + field( + raw, + "cmd_render.utile_width_px", + 156, + 1, + p.utile_width as u64, + ); + field( + raw, + "cmd_render.utile_height_px", + 157, + 1, + p.utile_height as u64, + ); + field(raw, "cmd_render.samples", 158, 1, p.samples as u64); + field( + raw, + "cmd_render.sample_size_B", + 159, + 1, + p.sample_size as u64, + ); + field( + raw, + "cmd_render.isp_merge_upper_x", + 160, + 4, + p.merge_upper_x as u64, + ); + field( + raw, + "cmd_render.isp_merge_upper_y", + 164, + 4, + p.merge_upper_y as u64, + ); + field(raw, "cmd_render.bg.usc", 168, 4, p.background.usc as u64); + field( + raw, + "cmd_render.bg.rsrc_spec", + 172, + 4, + (p.background.resource_spec & 0xffffffff) as u64, + ); + field( + raw, + "cmd_render.bg_rsrc_spec_hi", + 248, + 4, + (p.background.resource_spec >> 32) as u64, + ); + field(raw, "cmd_render.eot.usc", 176, 4, p.end_of_tile.usc as u64); + field( + raw, + "cmd_render.eot.rsrc_spec", + 180, + 4, + (p.end_of_tile.resource_spec & 0xffffffff) as u64, + ); + field( + raw, + "cmd_render.bg_eot_rsrc_spec_hi", + 240, + 4, + (p.end_of_tile.resource_spec >> 32) as u64, + ); + field( + raw, + "cmd_render.partial_bg.usc", + 184, + 4, + p.partial_background.usc as u64, + ); + field( + raw, + "cmd_render.partial_bg.rsrc_spec", + 188, + 4, + (p.partial_background.resource_spec & 0xffffffff) as u64, + ); + field( + raw, + "cmd_render.bg_partial_rsrc_spec_hi", + 252, + 4, + (p.partial_background.resource_spec >> 32) as u64, + ); + field( + raw, + "cmd_render.partial_eot.usc", + 192, + 4, + p.partial_end_of_tile.usc as u64, + ); + field( + raw, + "cmd_render.partial_eot.rsrc_spec", + 196, + 4, + (p.partial_end_of_tile.resource_spec & 0xffffffff) as u64, + ); + field( + raw, + "cmd_render.bg_eot_partial_rsrc_spec_hi", + 244, + 4, + (p.partial_end_of_tile.resource_spec >> 32) as u64, + ); + field( + raw, + "cmd_render.isp_bgobjdepth", + 200, + 4, + p.depth_clear as u64, + ); + field( + raw, + "cmd_render.isp_bgobjvals", + 204, + 4, + p.stencil_clear as u64, + ); + field( + raw, + "cmd_render.ts_vtx.start.handle", + 208, + 4, + p.vertex_timestamps.start.handle as u64, + ); + field( + raw, + "cmd_render.ts_vtx.start.offset", + 212, + 4, + p.vertex_timestamps.start.offset as u64, + ); + field( + raw, + "cmd_render.ts_vtx.end.handle", + 216, + 4, + p.vertex_timestamps.end.handle as u64, + ); + field( + raw, + "cmd_render.ts_vtx.end.offset", + 220, + 4, + p.vertex_timestamps.end.offset as u64, + ); + field( + raw, + "cmd_render.ts_frag.start.handle", + 224, + 4, + p.fragment_timestamps.start.handle as u64, + ); + field( + raw, + "cmd_render.ts_frag.start.offset", + 228, + 4, + p.fragment_timestamps.start.offset as u64, + ); + field( + raw, + "cmd_render.ts_frag.end.handle", + 232, + 4, + p.fragment_timestamps.end.handle as u64, + ); + field( + raw, + "cmd_render.ts_frag.end.offset", + 236, + 4, + p.fragment_timestamps.end.offset as u64, + ); +} + +fn check_compute(p: UapiComputeCommand, raw: &[u8]) { + field(raw, "cmd_compute.flags", 0, 4, p.flags as u64); + field( + raw, + "cmd_compute.sampler_count", + 4, + 4, + p.sampler_count as u64, + ); + field( + raw, + "cmd_compute.cdm_ctrl_stream_base", + 8, + 8, + p.control_stream_base as u64, + ); + field( + raw, + "cmd_compute.cdm_ctrl_stream_end", + 16, + 8, + p.control_stream_end as u64, + ); + field( + raw, + "cmd_compute.sampler_heap", + 24, + 8, + p.sampler_heap as u64, + ); + field( + raw, + "cmd_compute.helper.binary", + 32, + 4, + p.helper.binary as u64, + ); + field(raw, "cmd_compute.helper.cfg", 36, 4, p.helper.config as u64); + field(raw, "cmd_compute.helper.data", 40, 8, p.helper.data as u64); + field( + raw, + "cmd_compute.ts.start.handle", + 48, + 4, + p.timestamps.start.handle as u64, + ); + field( + raw, + "cmd_compute.ts.start.offset", + 52, + 4, + p.timestamps.start.offset as u64, + ); + field( + raw, + "cmd_compute.ts.end.handle", + 56, + 4, + p.timestamps.end.handle as u64, + ); + field( + raw, + "cmd_compute.ts.end.offset", + 60, + 4, + p.timestamps.end.offset as u64, + ); +} + +fn fixture() -> Vec { + std::fs::read(std::env::var("ASAHI_UAPI_FIXTURE").expect("C fixture path")).unwrap() +} + +#[test] +fn c_header_producer_matches_production_decoder() { + let offsets = layout(); + assert_eq!(offsets["header"].0, UAPI_COMMAND_HEADER_SIZE); + assert_eq!(offsets["attachment"].0, UAPI_ATTACHMENT_SIZE); + assert_eq!(offsets["timestamps"].0, 16); + assert_eq!(offsets["cmd_render.ts_frag_end_copies"].0, UAPI_RENDER_SIZE); + assert_eq!(offsets["cmd_compute.ts_end_copies"].0, UAPI_COMPUTE_SIZE); + let bytes = fixture(); + let render_offset = 2 * UAPI_COMMAND_HEADER_SIZE + UAPI_ATTACHMENT_SIZE; + let render_size = u16::from_le_bytes( + bytes[render_offset - 6..render_offset - 4] + .try_into() + .unwrap(), + ) as usize; + let compute_offset = render_offset + render_size + UAPI_COMMAND_HEADER_SIZE; + let mut parser = UapiCommandParser::new(&bytes); + let Some(ParsedHardwareCommand::Render { + index, + header, + payload, + vertex_attachments, + fragment_attachments, + }) = parser.next_hardware().unwrap() + else { + panic!("render missing") + }; + assert_eq!(index, 2); + assert_eq!( + header, + UapiCommandHeader { + command_type: 0, + size: render_size as u16, + render_barrier: UAPI_BARRIER_NONE, + compute_barrier: UAPI_BARRIER_NONE, + } + ); + check_render(payload, &bytes[render_offset..render_offset + render_size]); + assert_eq!(vertex_attachments.count, 1); + let attachment = vertex_attachments.as_slice()[0]; + field( + &bytes[8..32], + "attachment.pointer", + 0, + 8, + attachment.address, + ); + field(&bytes[8..32], "attachment.size", 8, 8, attachment.size); + assert!(fragment_attachments.as_slice().is_empty()); + let Some(ParsedHardwareCommand::Compute { + index, + header, + payload, + attachments, + }) = parser.next_hardware().unwrap() + else { + panic!("compute missing") + }; + assert_eq!(index, 3); + assert_eq!( + header, + UapiCommandHeader { + command_type: 1, + size: (bytes.len() - compute_offset) as u16, + render_barrier: UAPI_BARRIER_NONE, + compute_barrier: UAPI_BARRIER_NONE, + } + ); + check_compute(payload, &bytes[compute_offset..]); + assert!(attachments.as_slice().is_empty()); + assert!(parser.next_hardware().unwrap().is_none()); + assert_eq!(parser.finish(), Ok(2)); +} + +#[test] +fn short_payloads_zero_extend_but_missing_bytes_are_rejected() { + // Every length includes cuts within each multi-byte field, for both kinds. + for (kind, size) in [(0, 256), (1, 64)] { + let supplied: Vec = (0..size).map(|i| (i % 254 + 1) as u8).collect(); + for length in 0..=size { + let raw = &supplied[..length]; + let bytes = frame(kind, raw); + let mut parser = UapiCommandParser::new(&bytes); + match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Render { payload, .. } => check_render(payload, raw), + ParsedHardwareCommand::Compute { payload, .. } => check_compute(payload, raw), + } + assert!(parser.next_hardware().unwrap().is_none()); + assert_eq!(parser.finish(), Ok(1)); + for cut in 1..bytes.len() { + assert!(parse(&bytes[..cut]).is_err()); + } + } + } + assert_eq!(parse(&[]), Err(UapiTranslateError::HardwareCommandCount)); +} + +#[test] +fn named_optional_timestamp_copies_are_refused() { + for (kind, name) in [ + (0, "cmd_render.ts_frag_end_copies"), + (1, "cmd_compute.ts_end_copies"), + ] { + let (offset, size) = layout()[name]; + let mut raw = vec![0; offset + size]; + assert_eq!(parse(&frame(kind, &raw)), Ok(1)); + for i in offset..raw.len() { + raw[i] = 1; + assert_eq!( + parse(&frame(kind, &raw)), + Err(UapiTranslateError::PayloadTail), + "{name} byte {i}" + ); + raw[i] = 0; + } + } +} + +#[test] +fn unsupported_nonzero_extensions_never_disappear() { + for (kind, size) in [(0, UAPI_RENDER_SIZE), (1, UAPI_COMPUTE_SIZE)] { + for extra in [1, 8, 64, 128] { + let mut payload = vec![0; size + extra]; + assert_eq!(parse(&frame(kind, &payload)), Ok(1)); + for offset in size..payload.len() { + payload[offset] = 1; + assert_eq!( + parse(&frame(kind, &payload)), + Err(UapiTranslateError::PayloadTail) + ); + payload[offset] = 0; + } + } + } + assert_eq!( + parse(&frame(0xffff, &[])), + Err(UapiTranslateError::CommandType) + ); +} + +#[test] +fn attachment_shapes_counts_and_reserved_fields_are_checked() { + for kind in 2..=4 { + for size in 1..UAPI_ATTACHMENT_SIZE { + assert_eq!( + parse(&frame(kind, &vec![0; size])), + Err(UapiTranslateError::AttachmentPayloadSize) + ); + } + let mut bytes = frame(kind, &vec![0; UAPI_ATTACHMENT_SIZE * UAPI_MAX_ATTACHMENTS]); + bytes.extend(frame(1, &[])); + assert_eq!(parse(&bytes), Ok(1)); + assert_eq!( + parse(&frame( + kind, + &vec![0; UAPI_ATTACHMENT_SIZE * (UAPI_MAX_ATTACHMENTS + 1)] + )), + Err(UapiTranslateError::AttachmentCount) + ); + for offset in 16..24 { + let mut payload = [0; UAPI_ATTACHMENT_SIZE]; + payload[offset] = 1; + assert_eq!( + parse(&frame(kind, &payload)), + Err(UapiTranslateError::AttachmentFields) + ); + } + let mut bytes = frame(kind, &[]); + bytes[4..6].copy_from_slice(&0u16.to_le_bytes()); + assert_eq!(parse(&bytes), Err(UapiTranslateError::AttachmentBarrier)); + } +} + +#[test] +fn attachment_lists_reach_their_commands() { + let entry = |pointer: u64, size: u64| { + let mut raw = [0u8; UAPI_ATTACHMENT_SIZE]; + raw[..8].copy_from_slice(&pointer.to_le_bytes()); + raw[8..16].copy_from_slice(&size.to_le_bytes()); + raw + }; + let vertex = entry(0x1_2345_6000, 0x1_0000_0040).to_vec(); + let fragment = [ + entry(0x2_0000_8000, 0x80), + entry(0x3_0000_c000, 0x2_0000_0100), + ] + .concat(); + let compute = entry(0x4_0000_1000, 0x5_0000_0010).to_vec(); + let check = |list: UapiAttachmentList, raw: &[u8]| { + assert_eq!(list.as_slice().len(), raw.len() / UAPI_ATTACHMENT_SIZE); + for (attachment, raw) in list.as_slice().iter().zip(raw.chunks(UAPI_ATTACHMENT_SIZE)) { + field(raw, "attachment.pointer", 0, 8, attachment.address); + field(raw, "attachment.size", 8, 8, attachment.size); + } + }; + + // Lists persist until replaced; an empty list clears only its own kind. + let mut bytes = frame(2, &vertex); + bytes.extend(frame(3, &fragment)); + bytes.extend(frame(4, &compute)); + bytes.extend(frame(0, &[])); + bytes.extend(frame(1, &[])); + bytes.extend(frame(3, &[])); + bytes.extend(frame(0, &[])); + let mut parser = UapiCommandParser::new(&bytes); + for expected_fragment in [&fragment[..], &[]] { + let Some(ParsedHardwareCommand::Render { + vertex_attachments, + fragment_attachments, + .. + }) = parser.next_hardware().unwrap() + else { + panic!("render missing") + }; + check(vertex_attachments, &vertex); + check(fragment_attachments, expected_fragment); + if !expected_fragment.is_empty() { + let Some(ParsedHardwareCommand::Compute { attachments, .. }) = + parser.next_hardware().unwrap() + else { + panic!("compute missing") + }; + check(attachments, &compute); + } + } + assert!(parser.next_hardware().unwrap().is_none()); + assert_eq!(parser.finish(), Ok(3)); +} + +#[test] +fn dependencies_count_hardware_commands_and_bound_the_submit() { + for field in [4, 6] { + let mut bytes = frame(1, &[]); + bytes[field..field + 2].copy_from_slice(&1u16.to_le_bytes()); + assert_eq!(parse(&bytes), Err(UapiTranslateError::BarrierFuture)); + } + let mut bytes = frame(2, &[]); + bytes.extend(frame(0, &[])); + let mut compute = frame(1, &[]); + compute[4..6].copy_from_slice(&1u16.to_le_bytes()); + bytes.extend(compute); + assert_eq!(parse(&bytes), Ok(2)); + let command = frame(1, &[]); + assert_eq!( + parse(&command.repeat(UAPI_MAX_HARDWARE_COMMANDS as usize)), + Ok(UAPI_MAX_HARDWARE_COMMANDS) + ); + assert_eq!( + parse(&command.repeat(UAPI_MAX_HARDWARE_COMMANDS as usize + 1)), + Err(UapiTranslateError::HardwareCommandCount) + ); +} + +struct Region { + start: u64, + end: u64, + writable: bool, +} +impl GpuAddressSpace for Region { + fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool { + let Some(end) = address.checked_add(size) else { + return false; + }; + address >= self.start && end <= self.end && (access == GpuAccess::Read || self.writable) + } +} + +#[test] +fn attachment_validation_rejects_readonly_zero_and_wrapped_ranges() { + let mut list = UapiAttachmentList::EMPTY; + list.count = 1; + let mut region = Region { + start: 0x4000, + end: 0x8000, + writable: false, + }; + list.entries[0] = UapiAttachment { + address: 0x4000, + size: 0x4000, + }; + assert_eq!( + validate_attachments(®ion, &list), + Err(UapiTranslateError::AddressRange) + ); + region.writable = true; + assert_eq!(validate_attachments(®ion, &list), Ok(())); + for (address, size) in [(0x4000, 0), (0x4000, 0x4001), (u64::MAX, 2)] { + list.entries[0] = UapiAttachment { address, size }; + assert_eq!( + validate_attachments(®ion, &list), + Err(UapiTranslateError::AddressRange) + ); + } +} + +#[test] +fn usc_windows_reject_misalignment_overflow_and_unmapped_programs() { + let queue = QueueUscWindow { + base: USC_WINDOW_SIZE, + user_start: 0x4000, + user_end: 2 * USC_WINDOW_SIZE, + }; + assert_eq!(queue.validate(), Ok(())); + assert_eq!( + QueueUscWindow { + base: queue.base + 1, + ..queue + } + .validate(), + Err(UapiTranslateError::QueueUscAlignment) + ); + assert_eq!( + QueueUscWindow { + base: u64::MAX & !(USC_WINDOW_SIZE - 1), + ..queue + } + .validate(), + Err(UapiTranslateError::QueueUscRange) + ); + assert_eq!( + QueueUscWindow { + user_end: queue.user_end - 1, + ..queue + } + .validate(), + Err(UapiTranslateError::QueueUscRange) + ); + let region = Region { + start: queue.base + 8, + end: queue.base + 12, + writable: false, + }; + assert_eq!(queue.full_program(®ion, 9), Ok(queue.base + 8)); + assert_eq!( + queue.full_program(®ion, 0), + Err(UapiTranslateError::ProgramOffset) + ); + assert_eq!( + queue.full_program(®ion, 16), + Err(UapiTranslateError::AddressRange) + ); +} + +#[test] +fn unsupported_firmware_cannot_reach_the_existing_profile() { + let identity = (1, 1, 0x07022000, [0xff, 0, 0]); + assert!(admission::validate_identity( + &[14, 8, 3], + identity.0, + identity.1, + identity.2, + identity.3 + ) + .is_some()); + for version in [&[26, 6, 2][..], &[14, 8][..], &[14, 8, 3, 0][..], &[][..]] { + assert!(admission::validate_identity( + version, identity.0, identity.1, identity.2, identity.3 + ) + .is_none()); + } +} + +#[test] +fn firmware_profile_agrees_with_the_actual_c_header() { + assert_eq!( + layout()["profile.params_required"].0 as u64, + m3_profile::PARAMS_REQUIRED + ); + assert_eq!(layout()["profile.vm_create_size"].0, 24); + assert_eq!( + layout()["profile.hal200"].0 as u32, + m3_profile::Profile::Extended.hal() + ); + assert_eq!(layout()["profile.features"].0, 0x0a); +} diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index c4c6e21383fbf..04a2670f39b6c 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,7 +53,9 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'test-m3-uapi.py': [([], PASS)], 'test-m3-mailbox.py': [([], PASS)], + 'test-j613-dcp-power.py': [([], PASS)], 'aop-als-channel-tests.py': [([], PASS)], 'mtp-pm-tests.py': [(out(), PASS), (out() + ['--old-transport', '4db3a5a22c43'], PASS)], @@ -171,6 +173,7 @@ def mutations(flag, names, base, expect=KILL): # Production G17 decoder vectors also build directly with rustc --test. ASAHI_RUST.append('g17-telemetry-tests.rs') +ASAHI_RUST.append('m3-firmware-tests.rs') # Python that runs on the host but imports its neighbours, so it keeps the # script directory on sys.path (-E -s instead of -I). diff --git a/tools/asahi/test-j613-dcp-power.py b/tools/asahi/test-j613-dcp-power.py new file mode 100644 index 0000000000000..3bc1a3379c253 --- /dev/null +++ b/tools/asahi/test-j613-dcp-power.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production J613 PMGR predicate and test PMGR fields and unrelated bits.""" +from pathlib import Path +import os +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +source = (root / "drivers/gpu/drm/apple/j613_dcp_activate.c").read_text() +start = source.index("static bool j613_power_state_active(") +end = source.index("\n}\n", start) + 3 +stub = r""" +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +#define GENMASK(a,b) (((1U << ((a)-(b)+1))-1) << (b)) +#define FIELD_GET(mask,v) (((v) & (mask)) >> __builtin_ctz(mask)) +""" +main = r""" +int main(void) { + /* Only ACTUAL[7:4], TARGET[3:0] and AUTO_ENABLE[28] affect readiness. */ + const u32 unrelated_mask = ~(GENMASK(7, 0) | BIT(28)); + u32 unrelated[32] = {0, unrelated_mask, + 0xaaaaaaaaU & unrelated_mask, + 0x55555555U & unrelated_mask}; + unsigned int count = 4; + for (unsigned int bit = 8; bit < 32; bit++) + if (bit != 28) unrelated[count++] = BIT(bit); + for (unsigned int i = 0; i < count; i++) + for (u32 actual = 0; actual < 16; actual++) + for (u32 target = 0; target < 16; target++) + for (u32 automatic = 0; automatic < 2; automatic++) { + u32 word = unrelated[i] | (actual << 4) | target | (automatic << 28); + bool expected = actual == 15 || (target == 15 && automatic); + if (j613_power_state_active(word) != expected) { + fprintf(stderr, "PMGR state 0x%08x: expected active=%d\n", word, expected); + return 1; + } + } + if (!j613_power_state_active(0x1404024f)) return 2; + if (j613_power_state_active(0x0404024f)) return 3; + if (j613_power_state_active(0x14040240)) return 4; + puts("PASS: 13824 PMGR combinations and captured-state regression controls"); + return 0; +} +""" +with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "power.c" + path.write_text(stub + source[start:end] + main) + for flags in (["-O0"], ["-O2"]): + exe = str(path.with_suffix("")) + subprocess.run([os.environ.get("CC", "cc"), "-Wall", "-Wextra", "-Werror", + *flags, str(path), "-o", exe], check=True) + subprocess.run([exe], check=True) diff --git a/tools/asahi/test-m3-mailbox.py b/tools/asahi/test-m3-mailbox.py index 2320d17f10f6f..167b0566b2e5d 100644 --- a/tools/asahi/test-m3-mailbox.py +++ b/tools/asahi/test-m3-mailbox.py @@ -77,6 +77,17 @@ def vec(values): assert!(!ok(&compat,®,&names,&irqs,1)); let mut v=names.clone(); v[0]^=1; assert!(!ok(&compat,®,&v,&irqs,cells)); for v in [b"apple,asc-mailbox-v4\\0".to_vec(), b"apple,t8122-agx-asc-mailbox\\0unknown\\0".to_vec(), b"apple,t6031-agx-asc-mailbox\\0".to_vec()] { assert!(!ok(&v,®,&names,&irqs,cells)); } + // Compatible lists are ordered: reversing either supported pair must fail. + for pair in soc.mailbox_compatibles { + let parts: Vec<_> = pair.split(|byte| *byte == 0).filter(|part| !part.is_empty()).collect(); + if parts.len() == 2 { + let mut reversed = parts[1].to_vec(); + reversed.push(0); + reversed.extend_from_slice(parts[0]); + reversed.push(0); + assert!(!ok(&reversed,®,&names,&irqs,cells)); + } + } for v in soc.mailbox_compatibles { assert!(ok(&v.to_vec(),®,&names,&irqs,cells)); } } fn main() { diff --git a/tools/asahi/test-m3-uapi.py b/tools/asahi/test-m3-uapi.py new file mode 100755 index 0000000000000..a63249e18869d --- /dev/null +++ b/tools/asahi/test-m3-uapi.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile public C producers and run the production Rust parser offline.""" +import argparse +import hashlib +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +def run(command, **kwargs): + subprocess.run(command, check=True, **kwargs) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mesa", type=Path, help="matched public Mesa source") + args = parser.parse_args() + root = Path(__file__).resolve().parents[2] + if sys.byteorder != "little": + raise SystemExit("C fixtures currently require a little-endian host") + cc = os.environ.get("CC", "cc") + rustc = os.environ.get("RUSTC", "rustc") + fixture_source = root / "tools/asahi/m3-uapi-fixture.c" + with tempfile.TemporaryDirectory(prefix="asahi-uapi-") as temporary: + work = Path(temporary) + headers = work / "headers" + run(["make", "-s", "-C", str(root), "O=" + str(work / "headers-build"), + "ARCH=arm64", "headers_install", "INSTALL_HDR_PATH=" + str(headers)]) + sources = [(headers / "include", "drm/asahi_drm.h")] + if args.mesa: + sources.append((args.mesa.resolve() / "include", "drm-uapi/asahi_drm.h")) + fixtures = [] + layouts = [] + prefixes = [] + for index, (includes, header) in enumerate(sources): + producer = work / ("producer-" + str(index)) + run([cc, "-std=c11", "-Wall", "-Wextra", "-Werror", + "-I" + str(includes), '-DASAHI_HEADER="' + header + '"', + str(fixture_source), "-o", str(producer)]) + prefix = work / ("prefix-" + str(index)) + with prefix.open("wb") as output: + run([str(producer)], stdout=output) + prefixes.append(prefix) + layout = work / ("layout-" + str(index)) + with layout.open("wb") as output: + run([str(producer), "--layout"], stdout=output) + layouts.append(layout) + fixture = work / ("fixture-" + str(index)) + with fixture.open("wb") as output: + run([str(producer), "--full"], stdout=output) + fixtures.append(fixture) + reference = prefixes[0].read_bytes() + if any(prefix.read_bytes() != reference for prefix in prefixes[1:]): + raise SystemExit("kernel/Mesa canonical fixture mismatch") + print("canonical fixture sha256=" + hashlib.sha256(reference).hexdigest(), flush=True) + for optimized in (False, True): + binary = work / ("tests-optimized" if optimized else "tests-debug") + command = [rustc, "--edition=2021", "--test", + str(root / "tools/asahi/m3-uapi-tests.rs"), "-o", str(binary)] + if optimized: + command.append("-O") + run(command) + for index, fixture in enumerate(prefixes + fixtures): + env = dict(os.environ, ASAHI_UAPI_FIXTURE=str(fixture), + ASAHI_UAPI_LAYOUT=str(layouts[index % len(layouts)])) + run([str(binary)], env=env) + + +if __name__ == "__main__": + main() From 32ad46810d2a07ffec5f7c1aa675bdb2b772024d Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:38:59 -0400 Subject: [PATCH 19/37] drm/asahi: construct and publish extended T8122 initialization data Build extended initialization owners, controller images and channel records with checked allocation spans. Publish the bounded loader table through existing M3 ownership. Keep extended runtime admission closed. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_channel_format.rs | 182 ++++++++ drivers/gpu/drm/asahi/m3/m3_compute_layout.rs | 171 ++++++-- drivers/gpu/drm/asahi/m3/m3_config.rs | 179 +++++++- drivers/gpu/drm/asahi/m3/m3_firmware.rs | 59 +++ drivers/gpu/drm/asahi/m3/m3_init_26.rs | 22 +- drivers/gpu/drm/asahi/m3/m3_mmio_26.rs | 33 +- drivers/gpu/drm/asahi/m3/m3_power_26.rs | 414 ++++++++++++++++++ drivers/gpu/drm/asahi/m3/mod.rs | 5 + tools/asahi/m3-runtime-tests.rs | 12 + tools/asahi/run-host-tests.py | 1 + tools/asahi/test-m3-power.py | 173 ++++++++ 11 files changed, 1215 insertions(+), 36 deletions(-) create mode 100644 drivers/gpu/drm/asahi/m3/m3_channel_format.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_power_26.rs create mode 100644 tools/asahi/test-m3-power.py diff --git a/drivers/gpu/drm/asahi/m3/m3_channel_format.rs b/drivers/gpu/drm/asahi/m3/m3_channel_format.rs new file mode 100644 index 0000000000000..45eebda112b73 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_channel_format.rs @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded firmware channel records. Publication and retirement belong to Config. + +use super::m3_init_layout::Error; + +pub(crate) const CONTROL_BYTES: usize = 64; +pub(crate) const RECEIVE_BYTES: usize = 72; +pub(crate) const SLOTS: u32 = 256; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum Action { + Initialize, + Register { + pipe: u32, + engine: u32, + }, + Idle { + enabled: bool, + }, + Release { + group: u8, + index: u8, + weight: u8, + address: u64, + }, +} + +impl Action { + /// The first two actions precede the first generation-bearing publication. + pub(crate) fn generation(self, published: u32, consumed: u32) -> Result { + match self { + Self::Initialize | Self::Register { .. } => { + if published != 0 || consumed != 0 { + return Err(Error::Address); + } + Ok(0) + } + _ => published.max(consumed).checked_add(1).ok_or(Error::Address), + } + } + + pub(crate) fn encode(self, output: &mut [u8]) -> Result<(), Error> { + let mut words = [0u32; 3]; + let mut release = None; + match self { + Self::Initialize => words[0] = 0x16, + Self::Register { pipe, engine } => { + if pipe >= 3 || !(1..=2).contains(&engine) { + return Err(Error::Address); + } + words = [0x1b, pipe, engine]; + } + Self::Idle { enabled } => words = [0x0a, enabled.into(), 0], + Self::Release { + group, + index, + weight, + address, + } => { + if index == 255 + || weight == 255 + || address < 0xffff_fc00_0000_0000 + || address & 7 != 0 + { + return Err(Error::Address); + } + words[0] = 0x14; + words[2] = u32::from_le_bytes([group, index, weight, 4]); + release = Some(address); + } + } + let bytes = output.get_mut(..CONTROL_BYTES).ok_or(Error::Size)?; + bytes.fill(0); + for (index, word) in words.iter().enumerate() { + bytes[index * 4..index * 4 + 4].copy_from_slice(&word.to_le_bytes()); + } + if let Some(address) = release { + bytes[12..20].copy_from_slice(&address.to_le_bytes()); + } + Ok(()) + } +} + +/// An empty ring is required before publishing a synchronous control transaction. +/// Both consumers must retire it before the next control action may be published. +pub(crate) fn idle_cursor(cursors: [u32; 3]) -> Result { + if cursors.iter().any(|value| *value >= SLOTS) { + return Err(Error::Bounds); + } + if cursors[0] != cursors[2] || cursors[1] != cursors[2] { + return Err(Error::Address); + } + Ok(cursors[2]) +} + +pub(crate) fn payload_offset(thread: usize, slot: u32) -> Result { + if thread >= 9 { + return Err(Error::Bounds); + } + Ok((thread * 256 + (slot & 255) as usize) * 0xd8) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn initialization_records_have_only_their_defined_words() { + for action in [Action::Initialize, Action::Register { pipe: 2, engine: 1 }] { + let mut output = [0xa5; CONTROL_BYTES + 8]; + action.encode(&mut output).unwrap(); + assert_eq!(&output[12..CONTROL_BYTES], &[0; CONTROL_BYTES - 12]); + assert_eq!(&output[CONTROL_BYTES..], &[0xa5; 8]); + assert_eq!(action.generation(0, 0), Ok(0)); + assert!(action.generation(1, 0).is_err()); + } + } + + #[test] + fn invalid_actions_preserve_the_entire_output() { + for action in [ + Action::Register { pipe: 3, engine: 1 }, + Action::Register { pipe: 0, engine: 0 }, + Action::Release { + group: 0, + index: 255, + weight: 1, + address: 0xffff_fc21_0000_0000, + }, + Action::Release { + group: 0, + index: 0, + weight: 1, + address: 8, + }, + ] { + let mut output = [0xa5; CONTROL_BYTES]; + assert!(action.encode(&mut output).is_err()); + assert_eq!(output, [0xa5; CONTROL_BYTES]); + } + } + + #[test] + fn generations_follow_consumption_and_never_wrap() { + let action = Action::Idle { enabled: true }; + assert_eq!(action.generation(3, 5), Ok(6)); + assert_eq!(action.generation(5, 3), Ok(6)); + assert!(action.generation(u32::MAX, 0).is_err()); + assert!(action.generation(0, u32::MAX).is_err()); + let mut output = [0; CONTROL_BYTES]; + action.encode(&mut output).unwrap(); + assert_eq!(&output[..8], &[10, 0, 0, 0, 1, 0, 0, 0]); + } + + #[test] + fn both_consumers_and_all_indices_are_checked() { + assert_eq!(idle_cursor([255, 255, 255]), Ok(255)); + for cursors in [[0, 1, 1], [1, 0, 1], [256, 256, 256]] { + assert!(idle_cursor(cursors).is_err()); + } + assert_eq!(payload_offset(8, 0x101), Ok((8 * 256 + 1) * 0xd8)); + assert!(payload_offset(9, 0).is_err()); + } + + #[test] + fn release_uses_the_packed_owned_pointer() { + let action = Action::Release { + group: 1, + index: 2, + weight: 3, + address: 0xffff_fc21_0400_0000, + }; + let mut output = [0; CONTROL_BYTES]; + action.encode(&mut output).unwrap(); + assert_eq!(&output[8..12], &[1, 2, 3, 4]); + assert_eq!( + u64::from_le_bytes(output[12..20].try_into().unwrap()), + 0xffff_fc21_0400_0000 + ); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs b/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs index 74bed858b71e2..64c8cb005c4a1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT use super::{m3_pool_layout::GpuRegion, m3_queue_layout::{Error, FirmwareVa}}; pub(crate) const SIZE: usize = 0x893; +pub(crate) const EXTENDED_SIZE: usize = 0x8b0; pub(crate) const REGISTERS: usize = 0x20; pub(crate) const REGISTER_COUNT: usize = 22; /// G15G takes the same list without its last two registers (0xa599, 0xd411). @@ -64,12 +65,22 @@ impl Command { /// Validate before any output mutation. Encoding operates on ordinary host /// bytes; the caller copies them into the owned unpublished command buffer. pub(crate) fn encode(&self, out: &mut [u8]) -> Result<(), Error> { - if out.len() < SIZE { return Err(Error::Size); } + self.encode_for(super::m3_profile::Profile::Legacy, 0, 0, out) + } + + /// Firmware envelopes vary independently of the GPU's CDM stream format. + pub(crate) fn encode_for(&self, profile: super::m3_profile::Profile, + asid_generation: u8, event_slot: u8, out: &mut [u8]) -> Result<(), Error> { + let extended = profile == super::m3_profile::Profile::Extended; + let size = if extended { EXTENDED_SIZE } else { SIZE }; + if out.len() < size { return Err(Error::Size); } + if extended && (self.register_set != RegisterSet::G15G || event_slot >= 4 + || (self.context != 0 && asid_generation == 0)) { return Err(Error::Address); } if self.context >= 64 || self.cdm_last.get() < self.cdm.get() { return Err(Error::Address); } if self.preemption.size() < 0x14a0 || self.microsequence_size == 0 || self.microsequence_size > 4096 { return Err(Error::Size); } - self.gpu_alias.add((SIZE - 3) as u64)?; // inclusive final aligned dword + self.gpu_alias.add(((size - 1) & !3) as u64)?; // inclusive final aligned dword let registers = self.gpu_alias.add(REGISTERS as u64)?; let preemption = self.preemption.base(); let values = [ @@ -82,45 +93,143 @@ impl Command { (0x1a0b9, 0), (0x1a0c1, 0), (0x101d1, 0), (0xd479, 0), (0x1a0e9, 8), (0x107a1, 0xff0000), (0xa599, 0x13200400020), (0xd411, 0x200000001), ]; - let bytes = &mut out[..SIZE];bytes.fill(0); + let bytes = &mut out[..size];bytes.fill(0); word(bytes, 0, 3); // compute opcode - pointer(bytes, COUNTER, u64::from(self.counter)); - word(bytes, CONTEXT, self.context); - pointer(bytes, 0x14, self.notifier.get()); - let count = match self.register_set { + if extended { + word(bytes, 0xc, self.context); + pointer(bytes, 0x10, self.notifier.get()); + } else { + pointer(bytes, COUNTER, u64::from(self.counter)); + word(bytes, CONTEXT, self.context); + pointer(bytes, 0x14, self.notifier.get()); + } + let legacy_count = match self.register_set { RegisterSet::G15S => REGISTER_COUNT, RegisterSet::G15G => G15G_REGISTER_COUNT, }; - for (n, (register, value)) in values.into_iter().take(count).enumerate() { - word(bytes, REGISTERS+n*REGISTER_STRIDE, register); - pointer(bytes, REGISTERS+n*REGISTER_STRIDE+4, value); + let mut count = 0; + let mut emit = |register, value| { + word(bytes, REGISTERS + count * REGISTER_STRIDE, register); + pointer(bytes, REGISTERS + count * REGISTER_STRIDE + 4, value); + count += 1; + }; + let limit = if extended { REGISTER_COUNT } else { legacy_count }; + for (register, value) in values.into_iter().take(limit) { + if extended && register == 0x12091 { continue; } + emit(register, value); + } + if extended { + emit(0x1a540, 1); + emit(0x014a9, 1); + emit(0x0a351, 1); } pointer(bytes, 0x720, registers.get()); bytes[0x728..0x72a].copy_from_slice(&(count as u16).to_le_bytes()); bytes[0x72a..0x72c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes()); pointer(bytes, 0x760, self.microsequence.get()); word(bytes, 0x768, self.microsequence_size); - // ComputeInfo2 mirrors the preemption buffer, inclusive CDM end and - // qualified control word from the register list. - pointer(bytes, 0x76c+0x28, preemption); - pointer(bytes, 0x76c+0x30, self.cdm_last.get()); - pointer(bytes, 0x76c+0x58, QUALIFIED_COMPUTE_CONTROL); - word(bytes, 0x7cc+0x14, u32::MAX); // EncoderParams.unk_mask - pointer(bytes, META, self.stamp.get()); - pointer(bytes, META+8, self.fw_stamp.get()); - word(bytes, STAMP_VALUE, self.stamp_value); - word(bytes, META+0x14, 2); // qualified compute event slot - word(bytes, FLUSH_STAMPS, u32::from(self.flush_stamps)); - word(bytes, META+0x20, 0xc10000); // producer UUID - word(bytes, QUEUE_COUNT, self.counter); - pointer(bytes, TAIL+8, self.timestamps[0].get()); - pointer(bytes, TAIL+16, self.timestamps[1].get()); - pointer(bytes, TAIL+0x2e, self.pool.get()); - bytes[TAIL+0x36] = 1; // SharedCommandTail.unk_846 - pointer(bytes, TAIL+0x37, 0xad0000); - pointer(bytes, TAIL+0x3f, 0x1600000); - pointer(bytes, TAIL+0x47, self.tail_scratch.get()); - bytes[TAIL+0x4f] = 1; // SharedCommandTail.unk_85f + let info = if extended { 0x770 } else { 0x76c }; + pointer(bytes, info + 0x28, preemption); + pointer(bytes, info + 0x30, self.cdm_last.get()); + pointer(bytes, info + 0x58, QUALIFIED_COMPUTE_CONTROL); + word(bytes, if extended { 0x7e8 } else { 0x7e0 }, u32::MAX); + let meta = if extended { 0x800 } else { META }; + pointer(bytes, meta, self.stamp.get()); + pointer(bytes, meta + 8, self.fw_stamp.get()); + word(bytes, meta + 0x10, self.stamp_value); + word(bytes, meta + 0x14, 2); + if extended { word(bytes, meta + 0x18, event_slot.into()); } + word(bytes, meta + 0x1c, u32::from(self.flush_stamps)); + word(bytes, meta + 0x20, 0xc10000); + word(bytes, meta + 0x24, self.counter); + let tail = if extended { 0x82c } else { TAIL }; + pointer(bytes, tail + 8, self.timestamps[0].get()); + pointer(bytes, tail + 16, self.timestamps[1].get()); + pointer(bytes, tail + if extended { 0x3e } else { 0x2e }, self.pool.get()); + bytes[tail + if extended { 0x46 } else { 0x36 }] = 1; + pointer(bytes, tail + if extended { 0x47 } else { 0x37 }, + if extended { 0x1588000 } else { 0xad0000 }); + pointer(bytes, tail + if extended { 0x4f } else { 0x3f }, + if extended { 0xef8000 } else { 0x1600000 }); + pointer(bytes, tail + if extended { 0x57 } else { 0x47 }, self.tail_scratch.get()); + if extended { bytes[0x88b] = asid_generation; } + else { bytes[TAIL + 0x4f] = 1; } Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use super::super::m3_profile::Profile; + + fn fixture() -> Command { + let firmware = |n: u64| FirmwareVa::new(0xffff_fc21_8000_0000 + n * 0x4000).unwrap(); + Command { + context: 3, counter: 7, notifier: firmware(0), + preemption: GpuRegion::new(0x5000_0000, 0x4000).unwrap(), + cdm: GpuVa::new(0x6000_0000).unwrap(), + cdm_last: GpuVa::new(0x6000_0ffc).unwrap(), + gpu_alias: GpuVa::new(0x7000_0000).unwrap(), + microsequence: firmware(1), microsequence_size: 0x300, + stamp: firmware(2), fw_stamp: firmware(3), stamp_value: 0x700, + timestamps: [firmware(4), firmware(5)], pool: firmware(6), + tail_scratch: firmware(7), flush_stamps: true, + register_set: RegisterSet::G15G, + } + } + fn read32(b: &[u8], o: usize) -> u32 { + u32::from_le_bytes(b[o..o + 4].try_into().unwrap()) + } + fn read64(b: &[u8], o: usize) -> u64 { + u64::from_le_bytes(b[o..o + 8].try_into().unwrap()) + } + + #[test] + fn extended_envelope_preserves_stream_and_native_visibility_boundary() { + let c = fixture(); + let mut b = [0xa5; EXTENDED_SIZE + 16]; + c.encode_for(Profile::Extended, 9, 2, &mut b).unwrap(); + assert_eq!(read32(&b, 0xc), c.context); + assert_eq!(read64(&b, 0x10), c.notifier.get()); + assert_eq!(read32(&b, 0x728) & 0xffff, 24); + let count = (read32(&b, 0x728) & 0xffff) as usize; + let cdm = (0..count).find(|n| read32(&b, 0x20 + n * 12) == 0x1a420).unwrap(); + assert_eq!(read64(&b, 0x24 + cdm * 12), c.cdm.get()); + assert_eq!(read64(&b, 0x7a0), c.cdm_last.get()); + assert_eq!(read64(&b, 0x800), c.stamp.get()); + assert_eq!(read64(&b, 0x808), c.fw_stamp.get()); + assert_eq!(read32(&b, 0x818), 2); + assert_eq!(read32(&b, 0x81c), 1); + assert_eq!(read64(&b, 0x834), c.timestamps[0].get()); + assert_eq!(read64(&b, 0x86a), c.pool.get()); + assert_eq!(read64(&b, 0x883), c.tail_scratch.get()); + assert_eq!(b[0x88b], 9); + assert_eq!(&b[EXTENDED_SIZE..], &[0xa5; 16]); + } + + #[test] + fn extended_validation_never_mutates_a_rejected_envelope() { + for (generation, slot) in [(0, 0), (1, 4), (1, 255)] { + let mut b = [0xa5; EXTENDED_SIZE]; + assert!(fixture().encode_for(Profile::Extended, generation, slot, &mut b).is_err()); + assert_eq!(b, [0xa5; EXTENDED_SIZE]); + } + let mut b = [0xa5; EXTENDED_SIZE - 1]; + assert!(fixture().encode_for(Profile::Extended, 1, 0, &mut b).is_err()); + assert_eq!(b, [0xa5; EXTENDED_SIZE - 1]); + } + + #[test] + fn legacy_wrapper_keeps_legacy_geometry_and_register_order() { + let c = fixture(); + let mut b = [0xa5; EXTENDED_SIZE]; + c.encode(&mut b).unwrap(); + assert_eq!(read32(&b, CONTEXT), c.context); + assert_eq!(read64(&b, 0x14), c.notifier.get()); + assert_eq!(read32(&b, 0x728) & 0xffff, 20); + assert_eq!(read32(&b, REGISTERS + 8 * REGISTER_STRIDE), 0x12091); + assert_eq!(read64(&b, META), c.stamp.get()); + assert_eq!(&b[SIZE..], &[0xa5; EXTENDED_SIZE - SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index c4a44c4b4f3b0..67f4f963637f2 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -49,6 +49,8 @@ impl Records { pub(crate) struct Config { objects: KVec, + profile: super::m3_profile::Profile, + control_generation: u32, _iomaps: KVec, pub(crate) completed_events: u64, pstates: crate::m3::m3_adt_config::PstateWatch, @@ -140,7 +142,8 @@ impl Config { crate::m3::m3_adt_config::check_upload(dev.as_ref().as_ref(), firmware, contents.iomaps, &mut objects)?; agx_memory::publish(); let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?; - Ok(Self { objects, _iomaps: iomaps, completed_events: 0, + Ok(Self { objects, profile: firmware.profile(), control_generation: 0, + _iomaps: iomaps, completed_events: 0, pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal, request_top:contents.table.0,request_value:0,request_checked:false, request_check_at:None,request_writes:0,request_observed:0 }) @@ -261,6 +264,9 @@ impl Config { Ok(true) } pub(crate) fn ready(&mut self) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + return Ok(self.objects[RUNTIME_FLAGS].read_u32(0x14)? == 1); + } // NewRegionB+4 is an input initialized to 1, not an acknowledgement. // These are firmware-written transitions present in compute-15's // pre-submit snapshot; both start at zero in the exported schema. @@ -343,10 +349,38 @@ impl Config { agx_memory::publish(); Ok(next) } + /// Publish an extended-format synchronous control. The caller rings the + /// doorbell after this method and waits for both retirement cursors. + pub(crate) fn control_extended(&mut self, action: super::m3_channel_format::Action) -> Result { + use super::m3_channel_format as format; + if self.profile != super::m3_profile::Profile::Extended { return Err(EINVAL); } + let state = storage::DEVICE_CONTROL * 2; + let cursors = [self.objects[state].read_u32(0)?, + self.objects[state].read_u32(0x10)?, self.objects[state].read_u32(0x20)?]; + let cursor = format::idle_cursor(cursors).map_err(|_| EBUSY)?; + let consumed = self.objects[CONTROL_REGION].read_u32(0xeae8)?; + let generation = action.generation(self.control_generation, consumed).map_err(|_| EOVERFLOW)?; + let mut record = [0; format::CONTROL_BYTES]; + action.encode(&mut record).map_err(|_| EINVAL)?; + self.objects[state + 1].write(cursor as usize * format::CONTROL_BYTES, &record)?; + if generation != 0 { + self.objects[GLOBALS].write(0xe41, &generation.to_le_bytes())?; + } + agx_memory::publish(); + let next = (cursor + 1) % format::SLOTS; + self.objects[state].u32(0x20, next)?; + agx_memory::publish(); + self.control_generation = generation; + Ok(next) + } + pub(crate) fn control_done(&mut self, next: u32) -> Result { Ok(self.objects[storage::DEVICE_CONTROL*2].read_u32(0)? == next && self.objects[storage::DEVICE_CONTROL*2].read_u32(0x10)? == next) } pub(crate) fn drain(&mut self, dev: &driver::AsahiDrmDevice) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + return self.drain_extended(dev); + } // Event, trace, stats and six independent firmware log rings. for (state, ring, size, count, channels) in [(storage::EVENT*2,storage::EVENT*2+1,0x38,256,1), (storage::TRACE*2,storage::TRACE*2+1,0x38,512,1), @@ -392,4 +426,147 @@ impl Config { } Ok(()) } + fn drain_extended(&mut self, dev: &driver::AsahiDrmDevice) -> Result { + use super::m3_channel_format as format; + // Snapshot each producer once. New arrivals remain for the next drain; + // a corrupt cursor is rejected before reading its ring allocation. + for (channel, capacity, threads) in [(storage::FIRMWARE_LOG, 256, 9), + (storage::EVENT, 256, 1), (storage::TRACE, 512, 1), (storage::STATS, 256, 1)] { + let state = channel * 2; + for thread in 0..threads { + let at = thread * 0x30; + let mut read = self.objects[state].read_u32(at)?; + let write = self.objects[state].read_u32(at + 0x20)?; + if read >= capacity || write >= capacity { return Err(EIO); } + let before = read; + while read != write { + let offset = (thread * capacity as usize + read as usize) * format::RECEIVE_BYTES; + let mut record = [0; format::RECEIVE_BYTES]; + self.objects[state + 1].read(offset, &mut record)?; + let kind = u32::from_le_bytes(record[..4].try_into().map_err(|_| EIO)?); + if channel == storage::FIRMWARE_LOG { + let slot = u32::from_le_bytes(record[8..12].try_into().map_err(|_| EIO)?); + let payload = format::payload_offset(thread, slot).map_err(|_| EIO)?; + if crate::debug::debug_enabled(crate::debug::DebugFlags::KTraceCh) { + let mut text = [0; 200]; + self.objects[FWLOG_PAYLOAD].read(payload + 0x10, &mut text)?; + let len = text.iter().position(|value| *value == 0).unwrap_or(text.len()); + dev_info!(dev.as_ref(), "M3 firmware log[{}]: {}\n", thread, + core::str::from_utf8(&text[..len]).unwrap_or("")); + } + } else if channel == storage::EVENT { + if kind != 1 { + dev_err!(dev.as_ref(), "M3 firmware event {}: {:02x?}\n", kind, record); + return Err(EIO); + } + let flags = u32::from_le_bytes(record[4..8].try_into().map_err(|_| EIO)?); + self.completed_events = self.completed_events.saturating_add((flags & 7).count_ones().into()); + } else if (channel == storage::TRACE && kind != 5) + || (channel == storage::STATS && kind == 0x16) { + dev_err!(dev.as_ref(), "M3 firmware channel {} record {}: {:02x?}\n", channel, kind, record); + return Err(EIO); + } + read = (read + 1) % capacity; + } + if read != before { self.objects[state].u32(at, read)?; } + } + } + agx_memory::publish(); + Ok(()) + } + +} + +/// All extended initialization owners, retained as one construction transaction. +/// Nothing is published to RTKit by this constructor. A failed allocation drops +/// its mappings before backing storage and leaves no root visible to firmware. +pub(crate) struct ExtendedStorage { + pub(crate) objects: KVec, + pub(crate) iomaps: KVec, + pub(crate) descriptors: KVec, + pub(crate) maximum_khz: u32, +} + +impl ExtendedStorage { + pub(crate) fn new( + dev: &driver::AsahiDrmDevice, + uat: &mmu::Uat, + pwr: &super::hw::PwrConfig, + sram_uv: &[u32], + bases: super::m3_mmio_26::RegisterBases, + ) -> Result { + use super::{m3_init_26 as plan, m3_mmio_26 as io}; + let mut objects = KVec::new(); + for index in 0..plan::COUNT { + let entry = plan::allocation(index).map_err(|_| EINVAL)?; + let alias = match index { + plan::GLOBAL_POOL => Some((uat.kernel_lower_vm(), 0x70_0000_0000)), + plan::GLOBAL_PARAMETERS => Some((uat.kernel_lower_vm(), 0x70_0000_8000)), + _ => None, + }; + let object = if plan::cached(index) { + Buffer::at_prot(dev, uat.kernel_vm(), alias, Some(entry.address), entry.size, + prot::PROT_FW_PRIV_RW, prot::PROT_GPU_SHARED_RW)? + } else { + Buffer::at(dev, uat.kernel_vm(), alias, Some(entry.address), entry.size, entry.gpu_shared)? + }; + objects.push(object, GFP_KERNEL)?; + } + let mut descriptors = KVec::new(); + descriptors.resize(io::SLOTS * io::RECORD_BYTES, 0, GFP_KERNEL)?; + let mut iomaps = KVec::new(); + for slot in 0..io::SLOTS { + let mut region = None; + if let Some(area) = io::area(slot, bases).map_err(|_| EINVAL)? { + let address = 0xffff_fc23_0000_0000 + slot as u64 * 0x10_0000; + let access = if area.writable() { prot::PROT_FW_MMIO_RW } else { prot::PROT_FW_MMIO_RO }; + for (physical, bytes, offset) in area.banks() { + if bytes == 0 { continue; } + let mapping = uat.kernel_vm().map_io_m3(address + offset as u64, + physical.try_into()?, bytes, access)?; + iomaps.push(mapping, GFP_KERNEL)?; + } + region = Some(init::Region::new(address, area.mapped_bytes()).map_err(|_| EINVAL)?); + } + io::encode(slot, bases, region, + &mut descriptors[slot * io::RECORD_BYTES..(slot + 1) * io::RECORD_BYTES]) + .map_err(|_| EINVAL)?; + } + // Runtime+0x18 references Fender SRAM, rather than anonymous DMA RAM. + // Retain its normal shared mapping independently of its MMIO view. + let scratch_physical = bases.gpu.checked_add(0xd6_0000).ok_or(EOVERFLOW)?; + let scratch = uat.kernel_vm().map_io_m3(0xffff_fc22_0000_0000, + scratch_physical.try_into()?, 0x8000, prot::PROT_FW_SHARED_RW)?; + let scratch_region = init::Region::new(scratch.iova(), 0x8000).map_err(|_| EINVAL)?; + iomaps.push(scratch, GFP_KERNEL)?; + let mut regions = [init::Region::new(0xffff_fc21_0000_0000, 1).map_err(|_| EINVAL)?; plan::COUNT]; + for (index, object) in objects.iter().enumerate() { + regions[index] = init::Region::new(object.va(), object.size()).map_err(|_| EINVAL)?; + } + let graph = plan::PointerGraph { owners: ®ions, scratch: scratch_region, + pool_alias: objects[plan::GLOBAL_POOL].gpu_va()?, + parameter_alias: objects[plan::GLOBAL_PARAMETERS].gpu_va()? }; + let mut root = [0; 0xc8]; + let mut runtime = [0; plan::RUNTIME_SIZE]; + let mut control = KVec::new(); + control.resize(plan::allocation(CONTROL_REGION).map_err(|_| EINVAL)?.size, 0, GFP_KERNEL)?; + graph.root(&mut root).map_err(|_| EINVAL)?; + graph.runtime(&mut runtime).map_err(|_| EINVAL)?; + graph.firmware_control(&mut control).map_err(|_| EINVAL)?; + let images = super::m3_power_26::PowerImages::new(pwr, sram_uv, + objects[plan::SENSOR_VALUE].va(), objects[plan::RETENTION_STATE].va(), + agx_memory::TIMESTAMP_RANGE.start, &descriptors)?; + objects[storage::ROOT].write(0, &root)?; + objects[RUNTIME_POINTERS].write(0, &runtime)?; + objects[CONTROL_REGION].write(0, &control)?; + objects[HARDWARE_DATA].write(0, &images.hardware)?; + objects[plan::PERFORMANCE_CONTROL].write(0, &images.performance)?; + objects[GLOBALS_POWER].write(0, &images.power)?; + objects[GLOBALS].write(0, &images.globals)?; + // Fragment scheduler entries start unassigned; these are input sentinels. + objects[UNKNOWN_C1].u32(0xc18, u32::MAX)?; + objects[UNKNOWN_C1].u32(0xc30, u32::MAX)?; + agx_memory::publish(); + Ok(Self { objects, iomaps, descriptors, maximum_khz: images.maximum_khz }) + } } diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index c9aa45cc02864..6bc87a33aaa05 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -255,6 +255,51 @@ impl Firmware { } } +/// Only the admitted extended image has this host-owned boot descriptor slot. +fn loader_span(profile: super::m3_profile::Profile, data_bytes: u64, input_bytes: usize) + -> Option> { + if profile != super::m3_profile::Profile::Extended || input_bytes != 53 * 40 + || data_bytes != T8122_LAYOUT_26.data_size { return None; } + let start: usize = 0x12_28e0; + let end = start.checked_add(input_bytes)?; + if end as u64 > data_bytes { return None; } + Some(start..end) +} + +#[cfg(not(test))] +impl Firmware { + /// Call after RTKit application startup and before exposing the InitData root. + /// The protocol grants the host this slot only during that boot interval. + pub(crate) fn publish_io_table( + &self, + pdev: &kernel::platform::Device>, + descriptors: &[u8], + ) -> kernel::error::Result { + use kernel::{c_str, io::mem::{Mem, MemFlag}, prelude::*}; + let data = self.resources.regions[5]; + let span = loader_span(self.profile(), data.size, descriptors.len()).ok_or(EINVAL)?; + let node = pdev.as_ref().of_node().ok_or(ENODEV)?; + let resource = agx_resources::reserved_resource(&node, c_str!("fw-data"))?; + if resource.start() != data.base || resource.size() != data.size { return Err(EINVAL); } + // SAFETY: This exact admitted no-map reservation remains firmware-owned. + // Only its bounded host boot slot is accessed, before root publication. + let mapping = unsafe { Mem::try_new(resource, MemFlag::WC.into()) }?; + if span.end > mapping.size() { return Err(EINVAL); } + // Validate the whole slot before changing any byte. Unexpected prior + // contents indicate a protocol/ownership mismatch, not a patchable image. + for offset in span.clone() { + // SAFETY: span is bounded by the retained mapping above. + if unsafe { mapping.ptr().add(offset).read_volatile() } != 0 { return Err(EBUSY); } + } + for (offset, byte) in span.zip(descriptors.iter().copied()) { + // SAFETY: same bounded host slot, exclusive under the boot protocol. + unsafe { mapping.ptr().add(offset).write_volatile(byte) }; + } + crate::m3::agx_memory::publish(); + Ok(()) + } +} + #[cfg(not(test))] pub(crate) fn identify_loaded( pdev: &kernel::platform::Device>, @@ -527,4 +572,18 @@ mod tests { assert!(!newer.accepts_runtime(KNOWN_IMAGES_T8122[0].uuid, &[0; 32])); } + #[test] + fn loader_slot_is_exact_bounded_and_profile_specific() { + use super::super::m3_profile::Profile; + let length = 53 * 40; + assert_eq!(loader_span(Profile::Extended, 0x12c000, length), Some(0x1228e0..0x123128)); + for size in [0, 0x123127, 0x12bfff, 0x12c001, u64::MAX] { + assert!(loader_span(Profile::Extended, size, length).is_none()); + } + for bytes in [0, length - 1, length + 1, usize::MAX] { + assert!(loader_span(Profile::Extended, 0x12c000, bytes).is_none()); + } + assert!(loader_span(Profile::Legacy, 0x12c000, length).is_none()); + } + } diff --git a/drivers/gpu/drm/asahi/m3/m3_init_26.rs b/drivers/gpu/drm/asahi/m3/m3_init_26.rs index f08f16200b070..a86432b2bbe2b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_init_26.rs +++ b/drivers/gpu/drm/asahi/m3/m3_init_26.rs @@ -62,7 +62,7 @@ pub(crate) fn allocation(index: usize) -> Result { } pub(crate) fn cached(index: usize) -> bool { - matches!(index, 38 | 39 | 42 | 43 | 44 | PERFORMANCE_CONTROL) + matches!(index, 39 | PERFORMANCE_CONTROL) } pub(crate) struct PointerGraph<'a> { @@ -103,7 +103,10 @@ impl PointerGraph<'_> { } let scratch = self.scratch.at(0, 0x8000)?; let scratch_end = scratch.checked_add(0x8000).ok_or(Error::Address)?; - if scratch & 0x3fff != 0 || scratch_end > 0xffff_fc2f_fc00_0000 { + if scratch & 0x3fff != 0 + || scratch < 0xffff_fc20_0000_0000 + || scratch_end > 0xffff_fc2f_fc00_0000 + { return Err(Error::Address); } for (index, address) in addresses.iter().enumerate() { @@ -272,4 +275,19 @@ mod tests { assert!(graph(&owners).runtime(&mut output).is_err()); assert_eq!(output, [0xa5; RUNTIME_SIZE]); } + #[test] + fn only_atomic_controller_owners_are_firmware_cacheable() { + let selected: Vec<_> = (0..COUNT).filter(|index| cached(*index)).collect(); + assert_eq!(selected, vec![39, PERFORMANCE_CONTROL]); + } + + #[test] + fn scratch_requires_its_firmware_host_window() { + let owners = owners(0); + let mut value = graph(&owners); + value.scratch = Region::new(0xffff_fc10_0000_0000, 0x8000).unwrap(); + let mut output = [0xa5; RUNTIME_SIZE]; + assert_eq!(value.runtime(&mut output), Err(Error::Address)); + assert_eq!(output, [0xa5; RUNTIME_SIZE]); + } } diff --git a/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs b/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs index e4008f239c4a0..a17bc0ee399c3 100644 --- a/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs +++ b/drivers/gpu/drm/asahi/m3/m3_mmio_26.rs @@ -18,6 +18,10 @@ pub(crate) struct Area { } impl Area { + pub(crate) fn writable(self) -> bool { + self.writable + } + pub(crate) fn page_offset(self) -> usize { (self.physical & 0x3fff) as usize } @@ -78,6 +82,16 @@ pub(crate) fn area(slot: usize, bases: RegisterBases) -> Result, Er if physical == 0 || physical.checked_add(bytes as u64).ok_or(Error::Address)? > 1 << 42 { return Err(Error::Address); } + if split + && (physical & 0x3fff != 0 + || physical + .checked_add(0x200_0000) + .and_then(|v| v.checked_add((bytes / 2) as u64)) + .filter(|v| *v <= 1 << 42) + .is_none()) + { + return Err(Error::Address); + } Ok(Some(Area { physical, bytes, @@ -143,8 +157,7 @@ pub(crate) fn encode( mod tests { use super::*; - // Register identities already present in the public T8122 configuration; - // memory-controller bank geometry is shared with the public 25G83 encoder. + // Resource owners exercise subpage mappings and split memory-controller banks. const BASES: RegisterBases = RegisterBases { gpu: 0x2_9000_0000, interrupt: 0x2_d101_4000, @@ -262,4 +275,20 @@ mod tests { assert_eq!(output, [0xa5; RECORD_BYTES]); } } + #[test] + fn split_banks_require_page_alignment_and_both_fit_the_bus() { + for memory in [BASES.memory + 4, (1 << 42) - 0x100_0000] { + let mut bases = BASES; + bases.memory = memory; + let mut output = [0xa5; RECORD_BYTES]; + assert!(encode( + 3, + bases, + Some(Region::new(0xffff_fc21_0000_0000, 0x30000).unwrap()), + &mut output + ) + .is_err()); + assert_eq!(output, [0xa5; RECORD_BYTES]); + } + } } diff --git a/drivers/gpu/drm/asahi/m3/m3_power_26.rs b/drivers/gpu/drm/asahi/m3/m3_power_26.rs new file mode 100644 index 0000000000000..2619c24b4eb6a --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_power_26.rs @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Build the extended M3 controller images from admitted device-tree inputs. +//! These are constructor inputs. Firmware owns subsequent controller outputs. + +use super::{hw, initdata::PStateTables}; +use crate::{f32, float::F32}; +use kernel::prelude::*; + +struct Image(KVec); +impl Image { + fn new(size: usize) -> Result { + let mut bytes = KVec::new(); + bytes.resize(size, 0, GFP_KERNEL)?; + Ok(Self(bytes)) + } + fn word(&mut self, offset: usize, value: u32) -> Result { + self.0 + .get_mut(offset..offset.checked_add(4).ok_or(EOVERFLOW)?) + .ok_or(ERANGE)? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) + } + fn pointer(&mut self, offset: usize, value: u64) -> Result { + self.0 + .get_mut(offset..offset.checked_add(8).ok_or(EOVERFLOW)?) + .ok_or(ERANGE)? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) + } + fn float(&mut self, offset: usize, value: F32) -> Result { + let bits = value.to_bits(); + if bits & 0x7f80_0000 == 0x7f80_0000 { + return Err(EINVAL); + } + self.word(offset, bits) + } + fn reciprocal(&mut self, negative: usize, positive: usize, periods: u32) -> Result { + if periods == 0 { + return Err(EINVAL); + } + let alpha = f32!(1.0) / F32::from(periods); + self.float(negative, f32!(1.0) - alpha)?; + self.float(positive, alpha) + } +} + +pub(crate) struct PowerImages { + pub(crate) hardware: KVec, + pub(crate) performance: KVec, + pub(crate) power: KVec, + pub(crate) globals: KVec, + pub(crate) maximum_khz: u32, +} + +impl PowerImages { + /// `sram_uv` follows the same OPP iteration order as PwrConfig::perf_states. + /// No reference-bin voltage or experimental OPP-power estimate is substituted. + pub(crate) fn new( + pwr: &hw::PwrConfig, + sram_uv: &[u32], + sensor: u64, + retention: u64, + timestamp_base: u64, + io_records: &[u8], + ) -> Result { + let table = PStateTables::new(pwr, true, None, false)?; + let second = table.secondary.ok_or(EINVAL)?; + if table.len != 9 + || table.max != 8 + || sram_uv.len() != pwr.perf_states.len() + || pwr.perf_base_pstate == 0 + || pwr.perf_base_pstate > 8 + || !(1..=1000).contains(&pwr.power_sample_period) + || pwr.pwr_min_duty_cycle > 100 + || pwr.avg_power_min_duty_cycle > 100 + || io_records.len() != super::m3_mmio_26::SLOTS * super::m3_mmio_26::RECORD_BYTES + || [sensor, retention, timestamp_base] + .iter() + .any(|value| *value < 0xffff_fc00_0000_0000 || value & 7 != 0) + { + return Err(EINVAL); + } + let top = &pwr.perf_states[table.primary[8] as usize]; + let base = &pwr.perf_states[table.primary[pwr.perf_base_pstate as usize] as usize]; + if top.freq_hz != 1_338_000_000 || base.freq_hz >= top.freq_hz { + return Err(EINVAL); + } + let period = pwr.power_sample_period; + let ppm_periods = pwr.ppm_filter_time_constant_ms / period; + let average_periods = pwr.avg_power_filter_tc_ms / period; + if [ + ppm_periods, + average_periods, + pwr.pwr_filter_time_constant, + pwr.perf_filter_time_constant, + pwr.perf_filter_time_constant2, + pwr.avg_power_target_filter_tc, + pwr.se_filter_time_constant, + pwr.se_filter_time_constant_1, + ] + .contains(&0) + { + return Err(EINVAL); + } + let clocks = |count: u32| { + count + .checked_mul(period) + .and_then(|v| v.checked_mul(24_000)) + .ok_or(EOVERFLOW) + }; + let dt = F32::from(period) / f32!(1000.0); + let ceiling = 800; + let base_units = pwr.perf_base_pstate * 100; + // The profile controller budget is independent of the loader's experimental + // OPP power model. It is not a demand floor or a replacement voltage table. + let budget = 22_044u32; + let mut main = Image::new(0x2658)?; + let mut performance = Image::new(0x6578)?; + let mut power = Image::new(0x670)?; + let mut globals = Image::new(0xe48)?; + + main.pointer(0x28, timestamp_base)?; + main.0[0x640..0xe88].copy_from_slice(io_records); + main.pointer(0xe88, retention)?; + main.word(0xe90, 0x8122)?; + main.word(0xe94, 2)?; + main.word(0xe9c, 4)?; + main.word(0xea0, 1)?; + main.word(0xea4, 1)?; + main.word(0xeb8, 1)?; + main.word(0xec0, 1)?; + main.word(0xec8, 1)?; + main.word(0xed0, 24_000)?; + main.word(0xed4, 1)?; + main.word(0xed8, period)?; + main.word(0xee0, 1)?; + main.word(0xee4, 1)?; + main.word(0xee8, 1)?; + main.word(0xf04, 31)?; + main.pointer(0xf24, 4)?; + main.word(0xf34, 1)?; + // Firmware completion is polled through its normal bounded status path. + main.word(0xf38, 0)?; + main.word(0xf4c, 49)?; + main.word(0xf6c, 1)?; + main.word(0xf88, 10)?; + main.word(0xf8c, 3)?; + main.word(0xfac, 1)?; + main.word(0xfb8, 22)?; + main.word(0xfbc, 5)?; + main.word(0xfc0, 10)?; + main.word(0xfc4, 8)?; + main.word(0x1cd8, 8)?; + for state in 0..9 { + let primary = table.primary[state] as usize; + let secondary = second[state] as usize; + let item = &pwr.perf_states[primary]; + let auxiliary = &pwr.perf_states[secondary]; + if item.volt_mv.len() != 1 + || auxiliary.volt_mv.len() != 1 + || !(100..=1100).contains(&item.volt_mv[0]) + || !(100..=1100).contains(&auxiliary.volt_mv[0]) + || !(100_000..=1_100_000).contains(&sram_uv[primary]) + || !(100_000..=1_100_000).contains(&sram_uv[secondary]) + || sram_uv[primary].div_ceil(1000) < item.volt_mv[0] + || sram_uv[secondary].div_ceil(1000) < auxiliary.volt_mv[0] + { + return Err(EINVAL); + } + main.word(0xfc8 + state * 4, item.freq_hz / 1_000_000)?; + main.word(0x1808 + state * 4, auxiliary.freq_hz / 1_000_000)?; + main.float(0x1848 + state * 4, f32!(1.02))?; + for lane in 0..16 { + main.word(0x1008 + state * 64 + lane * 4, item.volt_mv[0])?; + main.word( + 0x1408 + state * 64 + lane * 4, + sram_uv[primary].div_ceil(1000), + )?; + } + main.word(0x1cdc + state * 4, auxiliary.freq_hz / 1_000_000)?; + if state != 0 { + main.word(0x1d1c + state * 64, auxiliary.volt_mv[0])?; + } + main.word(0x211c + state * 64, sram_uv[secondary].div_ceil(1000))?; + if state != 0 { + // Profile indices for the admitted eight-state ladder. + let core_profile = if state <= 3 { state } else { 2 * state - 3 }; + let auxiliary_profile = if state <= 4 { state } else { 2 * state - 4 }; + main.word(0x19c8 + state * 4, core_profile as u32)?; + main.word(0x1a08 + state * 4, auxiliary_profile as u32)?; + let boost = item.freq_hz.saturating_sub(base.freq_hz) / 1_000_000; + main.word( + 0x1908 + state * 4, + 100 * boost / ((top.freq_hz - base.freq_hz) / 1_000_000), + )?; + } + } + // The native relative-power calibration of each running firmware state. + // Encode only these semantic percentages, not an object-image overlay. + for (state, percent) in [15, 25, 36, 46, 58, 75, 90, 100].into_iter().enumerate() { + main.word(0x18c8 + (state + 1) * 4, percent)?; + } + main.word(0x2544, 4)?; + main.word(0x2548, 6)?; + main.word(0x254c, 3)?; + main.word(0x2550, 7)?; + main.word(0x2554, 7)?; + main.word(0x2568, 10)?; + main.word(0x2570, 1)?; + main.word(0x258c, 0x160000)?; + main.word(0x2594, 1)?; + main.word(0x2598, 1)?; + main.word(0x259c, 1)?; + main.word(0x25a4, 1)?; + main.word(0x25a8, 1)?; + main.word(0x25ac, 1)?; + main.0[0x25b4..0x25e4].fill(0xff); + main.word(0x2604, 1)?; + main.word(0x2634, 1)?; + + globals.word(0x24, 3000)?; + globals.word(0x78, 1)?; + globals.word(0x84, budget)?; + for offset in [0x88, 0x8c, 0x98] { + globals.word(offset, ceiling)?; + } + globals.word(0x998, 40)?; + globals.word(0x99c, 10)?; + globals.word(0x9a0, 250)?; + globals.word(0x9b8, 2)?; + globals.word(0x9bc, 40)?; + globals.word(0x9c0, 5)?; + globals.word(0x9c8, 40)?; + globals.word(0x9cc, 50)?; + globals.word(0xe28, 3)?; + + power.word(0x10, average_periods)?; + power.float(0x14, pwr.avg_power_ki_only * dt)?; + power.float(0x18, pwr.avg_power_kp)?; + power.word(0x1c, pwr.avg_power_min_duty_cycle)?; + power.word(0x20, pwr.avg_power_target_filter_tc)?; + power.word(0xc8, 11_000)?; + power.word(0xcc, 8_000)?; + power.float(0xd8, pwr.fast_die0_proportional_gain)?; + power.float(0xdc, pwr.fast_die0_integral_gain * dt)?; + power.word(0x12c, budget)?; + power.float(0x130, pwr.ppm_kp)?; + power.float(0x134, pwr.ppm_ki * dt)?; + power.word(0x14c, 1)?; + + performance.word(4, pwr.pwr_sample_period_aic_clks)?; + performance.word(8, pwr.pwr_sample_period_aic_clks)?; + performance.word(0x10, 4)?; + performance.float(0x14, f32!(1.0))?; + performance.word(0x2c, 1)?; + performance.word(0x30, 1)?; + // Leave the observed hardware state at zero until firmware writes it. + for offset in [0x40, 0x80, 0xb78, 0xb80, 0xbc0] { + performance.word(offset, base_units)?; + } + for offset in [0x44, 0x6c, 0x70, 0x84] { + performance.word(offset, 1)?; + } + for offset in [ + 0x48, 0x88, 0x9f4, 0x9f8, 0xa38, 0xaac, 0xaf0, 0xb7c, 0x221c, 0x22b4, 0x22b8, 0x22f8, + 0x2870, 0x29f8, 0x29fc, 0x2a3c, 0x2b88, 0x2c44, 0x2c88, + ] { + performance.word(offset, ceiling)?; + } + performance.word(0x50, 100)?; + performance.word(0x60, 4)?; + performance.float(0x64, f32!(1.0))?; + performance.word(0x90, 100)?; + performance.word(0x9b8, 625)?; + performance.reciprocal(0x9c4, 0x9cc, pwr.pwr_filter_time_constant)?; + performance.float(0x9d4, pwr.pwr_integral_gain)?; + performance.float(0x9dc, F32::from(pwr.pwr_integral_min_clamp))?; + performance.float(0x9e0, F32::from(budget))?; + performance.float(0x9e4, pwr.pwr_proportional_gain)?; + performance.float(0x9ec, -F32::from(ceiling) / F32::from(budget))?; + for offset in [0x9f0, 0xaa8, 0x22b0] { + performance.word(offset, pwr.pwr_min_duty_cycle)?; + } + for offset in [0xa04, 0xabc, 0x28c8, 0x28cc, 0x28d0, 0x2a08] { + performance.word(offset, budget)?; + } + performance.word(0xa70, ppm_periods.checked_mul(4).ok_or(EOVERFLOW)?)?; + performance.reciprocal(0xa7c, 0xa84, ppm_periods)?; + performance.float(0xa8c, pwr.ppm_ki * dt)?; + performance.float(0xa98, f32!(65536.0))?; + performance.float(0xa9c, pwr.ppm_kp)?; + performance.word(0xac8, pwr.ppm_filter_time_constant_ms)?; + performance.word(0xad0, clocks(ppm_periods)?)?; + performance.word(0xb28, pwr.perf_tgt_utilization)?; + performance.word(0xb30, pwr.perf_boost_min_util)?; + performance.word(0xb34, pwr.perf_boost_ce_step)?; + performance.word(0xb38, pwr.perf_reset_iters)?; + performance.word(0xb40, 6)?; + performance.word(0xb44, 1)?; + performance.word(0xb48, pwr.perf_filter_drop_threshold)?; + performance.reciprocal(0xb4c, 0xb54, pwr.perf_filter_time_constant)?; + performance.reciprocal(0xb50, 0xb58, pwr.perf_filter_time_constant2)?; + performance.float(0xb5c, pwr.perf_integral_gain)?; + performance.float(0xb60, pwr.perf_integral_gain2)?; + performance.float(0xb64, F32::from(pwr.perf_integral_min_clamp))?; + performance.float(0xb68, f32!(95.0))?; + performance.float(0xb6c, pwr.perf_proportional_gain)?; + performance.float(0xb70, pwr.perf_proportional_gain2)?; + performance.float(0xb74, F32::from(8 - pwr.perf_base_pstate) / f32!(0.95))?; + performance.word(0xb8c, pwr.perf_tgt_utilization)?; + performance.float(0x21f0, f32!(65536.0))?; + performance.float(0x21f4, F32::from(pwr.pwr_min_duty_cycle))?; + performance.float(0x21f8, F32::from(ceiling))?; + performance.word(0x21fc, 1338)?; + performance.float(0x2218, f32!(100.0))?; + performance.float(0x2224, f32!(0.8))?; + performance.float(0x2228, f32!(0.2))?; + performance.word(0x2268, 0x4248)?; + performance.word(0x2270, 8_000)?; + performance.word(0x2278, 4)?; + performance.float(0x228c, f32!(1.0))?; + performance.float(0x2294, pwr.fast_die0_integral_gain * dt)?; + performance.float(0x22a0, f32!(65536.0))?; + performance.float(0x22a4, pwr.fast_die0_proportional_gain)?; + performance.word(0x22c4, 11_000)?; + performance.word(0x28c0, 1)?; + performance.reciprocal(0x28dc, 0x28e0, pwr.avg_power_target_filter_tc)?; + performance.word( + 0x28e4, + pwr.avg_power_target_filter_tc + .checked_mul(4) + .ok_or(EOVERFLOW)?, + )?; + performance.word( + 0x28e8, + pwr.avg_power_target_filter_tc + .checked_mul(period) + .ok_or(EOVERFLOW)?, + )?; + performance.word(0x28ec, clocks(pwr.avg_power_target_filter_tc)?)?; + performance.word(0x29bc, average_periods.checked_mul(4).ok_or(EOVERFLOW)?)?; + performance.reciprocal(0x29c8, 0x29d0, average_periods)?; + performance.float(0x29d8, pwr.avg_power_ki_only * dt)?; + performance.float(0x29e4, f32!(65536.0))?; + performance.float(0x29e8, pwr.avg_power_kp)?; + performance.word(0x29f4, pwr.avg_power_min_duty_cycle)?; + performance.float(0x2a04, F32::from(budget))?; + performance.word(0x2a14, pwr.avg_power_filter_tc_ms)?; + performance.word(0x2a1c, clocks(average_periods)?)?; + performance.word(0x2b4c, 50)?; + performance.word(0x2b50, 1)?; + performance.reciprocal(0x2b58, 0x2b60, pwr.se_filter_time_constant)?; + performance.reciprocal(0x2b5c, 0x2b64, pwr.se_filter_time_constant_1)?; + performance.float(0x2b68, pwr.se_ki * dt)?; + performance.float(0x2b6c, pwr.se_ki_1 * dt)?; + performance.float(0x2b74, f32!(65536.0))?; + performance.float(0x2b78, pwr.se_kp)?; + performance.float(0x2b7c, pwr.se_kp_1)?; + performance.word(0x2b84, 100)?; + performance.word(0x2b8c, 100)?; + performance.word(0x2b98, pwr.se_target)?; + performance.word( + 0x2ba4, + pwr.se_filter_time_constant + .checked_mul(period) + .ok_or(EOVERFLOW)?, + )?; + performance.word( + 0x2ba8, + pwr.se_filter_time_constant_1 + .checked_mul(period) + .ok_or(EOVERFLOW)?, + )?; + performance.word(0x2bac, clocks(pwr.se_filter_time_constant)?)?; + performance.word(0x2bb4, clocks(pwr.se_filter_time_constant_1)?)?; + performance.float(0x2b94, f32!(8000.0))?; + for offset in [0x2bc0, 0x2bc4, 0x2c30] { + performance.float(offset, f32!(65536.0))?; + } + performance.word(0x2be8, 8000)?; + performance.word(0x2bec, 4)?; + performance.word(0x2bf0, pwr.se_inactive_threshold)?; + performance.word(0x2bf4, pwr.se_engagement_criteria as u32)?; + performance.word(0x2bf8, 2)?; + performance.word(0x2bfc, 4)?; + performance.word(0x2c00, pwr.se_reset_criteria)?; + performance.word(0x2c40, pwr.avg_power_min_duty_cycle)?; + performance.word(0x2e2c, 4)?; + performance.float(0x2e38, f32!(0.96875))?; + performance.float(0x2e40, f32!(0.03125))?; + performance.float(0x2e48, f32!(5.0))?; + performance.float(0x2e54, f32!(65536.0))?; + performance.float(0x2e58, f32!(15.0))?; + performance.word(0x2e64, 700)?; + for offset in [0x2e68, 0x2e6c, 0x2e78] { + performance.word(offset, ceiling)?; + } + performance.word(0x2f08, 0x4248)?; + performance.word(0x2f10, 125)?; + performance.word(0x2f14, 1)?; + performance.pointer(0x3b50, sensor)?; + performance.word(0x3b58, 0x4b48)?; + performance.word(0x3b60, 1)?; + performance.word(0x62a8, 1)?; + Ok(Self { + hardware: main.0, + performance: performance.0, + power: power.0, + globals: globals.0, + maximum_khz: top.freq_hz / 1000, + }) + } +} diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index 7073ecf294d63..d885da1a8647c 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -202,3 +202,8 @@ mod t8122_hwdata; mod m3_init_26; mod m3_profile; mod m3_mmio_26; + +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_power_26; + +mod m3_channel_format; diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index f1768a242147b..abac06668c0a1 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -24,3 +24,15 @@ mod m3_profile; #[path = "../../drivers/gpu/drm/asahi/m3/m3_mmio_26.rs"] mod m3_mmio_26; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_queue_layout.rs"] +mod m3_queue_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_pool_layout.rs"] +mod m3_pool_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_layout.rs"] +mod m3_compute_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_channel_format.rs"] +mod m3_channel_format; diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 04a2670f39b6c..5dfda83574ea6 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,6 +53,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'test-m3-power.py': [([], PASS)], 'test-m3-uapi.py': [([], PASS)], 'test-m3-mailbox.py': [([], PASS)], 'test-j613-dcp-power.py': [([], PASS)], diff --git a/tools/asahi/test-m3-power.py b/tools/asahi/test-m3-power.py new file mode 100644 index 0000000000000..51bea088cd9db --- /dev/null +++ b/tools/asahi/test-m3-power.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the production controller writer and soft-float code without a GPU. + +Kernel allocation/DT adapters are replaced by host adapters. Public production +power types and PStateTables are extracted as complete balanced Rust items, +never restated by hand. Their input hashes are printed with the test evidence. +""" +from pathlib import Path +import hashlib +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +M3 = ROOT / "drivers/gpu/drm/asahi/m3" + + +def item(source, anchor): + start = source.index(anchor) + opening = source.index("{", start) + depth = 1 + end = opening + 1 + while depth: + if source[end] == "{": + depth += 1 + elif source[end] == "}": + depth -= 1 + end += 1 + return source[start:end] + + +shim = r''' +#![allow(dead_code)] +extern crate self as kernel; +pub type Error = i32; +pub type Result = core::result::Result; +pub const EINVAL: Error = 22; +pub const EOVERFLOW: Error = 75; +pub const ERANGE: Error = 34; +pub const GFP_KERNEL: () = (); +pub unsafe trait Zeroable {} +#[derive(Debug)] +pub struct KVec(Vec); +impl Default for KVec { fn default() -> Self { Self(Vec::new()) } } +impl KVec { + pub fn new() -> Self { Self::default() } + pub fn resize(&mut self, n: usize, value: T, _: ()) -> Result where T: Clone { + self.0.resize(n, value); Ok(()) + } +} +impl core::ops::Deref for KVec { type Target = Vec; fn deref(&self) -> &Vec { &self.0 } } +impl core::ops::DerefMut for KVec { fn deref_mut(&mut self) -> &mut Vec { &mut self.0 } } +pub mod prelude { pub use crate::{Error,Result,EINVAL,EOVERFLOW,ERANGE,GFP_KERNEL,KVec,Zeroable}; } +pub mod of { + pub struct Property<'a>(pub &'a [u8]); + pub trait PropertyUnit: Sized { const UNIT_SIZE: usize; fn from_bytes(data: &[u8]) -> crate::Result; } + impl PropertyUnit for u32 { + const UNIT_SIZE: usize = 4; + fn from_bytes(data: &[u8]) -> crate::Result { + Ok(u32::from_be_bytes(data.try_into().map_err(|_| crate::EINVAL)?)) + } + } + impl TryFrom> for u32 { + type Error = crate::Error; + fn try_from(value: Property<'_>) -> crate::Result { Self::from_bytes(value.0) } + } +} +''' +fixture = r''' +#[cfg(test)] mod controller_tests { + use super::*; + use float::F32; + fn input() -> hw::PwrConfig { + let mut p = hw::PwrConfig::default(); + p.perf_states = KVec((0..9).map(|i| hw::PState { + freq_hz: if i == 8 { 1_338_000_000 } else { i * 100_000_000 }, + volt_mv: KVec(vec![if i == 0 { 125 } else { 450 + i * 50 }]), pwr_mw: i * 1000, + }).collect()); + p.perf_base_pstate = 1; + p.power_sample_period = 8; + p.ppm_filter_time_constant_ms = 16; + p.avg_power_filter_tc_ms = 16; + p.pwr_filter_time_constant = 2; + p.perf_filter_time_constant = 2; + p.perf_filter_time_constant2 = 2; + p.avg_power_target_filter_tc = 2; + p.se_filter_time_constant = 2; + p.se_filter_time_constant_1 = 2; + p.pwr_min_duty_cycle = 30; + p.avg_power_min_duty_cycle = 20; + p + } + fn build(p: &hw::PwrConfig, sram: &[u32]) -> Result { + m3_power_26::PowerImages::new(p, sram, 0xffff_fc21_0340_0000, + 0xffff_fc21_0350_0000, 0xffff_fc2f_fc00_0000, &[0; 53 * 40]) + } + fn word(b: &[u8], at: usize) -> u32 { u32::from_le_bytes(b[at..at + 4].try_into().unwrap()) } + #[test] fn native_controller_bounds_are_distinct_from_outputs_and_hardware_state() { + let image = build(&input(), &[950_001; 9]).unwrap(); + assert_eq!(image.hardware.len(), 0x2658); + assert_eq!(image.performance.len(), 0x6578); + assert_eq!(image.power.len(), 0x670); + assert_eq!(image.globals.len(), 0xe48); + assert_eq!(image.maximum_khz, 1_338_000); + assert_eq!(word(&image.performance, 0x2c), 1); + assert_eq!(word(&image.performance, 0x30), 1); + assert_eq!(word(&image.performance, 0x38), 0); + assert_eq!(word(&image.performance, 0x9f0), 30); + assert_eq!(word(&image.performance, 0xa38), 800); + assert_eq!(word(&image.performance, 0xb78), 100); + assert_eq!(word(&image.performance, 0xb7c), 800); + assert_eq!(word(&image.hardware, 0x1408 + 64), 951); + assert_eq!(word(&image.performance, 0x2268), 0x4248); + assert_eq!(word(&image.performance, 0x3b58), 0x4b48); + } + #[test] fn malformed_tunables_and_nonfinite_gains_are_rejected() { + for period in [0, 1001, u32::MAX] { + let mut p = input(); p.power_sample_period = period; + assert!(build(&p, &[950_000; 9]).is_err()); + } + for bits in [0x7f800000, 0xff800000, 0x7fc00001] { + let mut p = input(); p.pwr_integral_gain = F32::from_bits(bits); + assert!(build(&p, &[950_000; 9]).is_err()); + } + assert!(build(&input(), &[950_000; 8]).is_err()); + assert!(build(&input(), &[100_000; 9]).is_err()); + let mut p = input(); p.avg_power_target_filter_tc = u32::MAX; + assert!(build(&p, &[950_000; 9]).is_err()); + } + #[test] fn dt_order_is_not_the_firmware_voltage_order() { + let p = input(); + let expected = build(&p, &[950_000; 9]).unwrap(); + let mut shuffled = input(); + shuffled.perf_states.swap(2, 6); + let actual = build(&shuffled, &[950_000; 9]).unwrap(); + assert_eq!(actual.hardware.0, expected.hardware.0); + assert_eq!(actual.performance.0, expected.performance.0); + } +} +''' + + +def main(): + hw = (M3 / "hw.rs").read_text() + init = (M3 / "initdata.rs").read_text() + code = shim + # Keep the production soft-float prefix verbatim. Its unrelated existing + # print-only TODO test lacks an import and is not part of this harness. + soft_float = (ROOT / "drivers/gpu/drm/asahi/float.rs").read_text() + code += "mod float {\n" + soft_float.split("// TODO: Make this an actual test", 1)[0] + "\n}\n" + code += "mod hw { use crate::{KVec,float::F32};\n" + for name in ["PState", "PowerZone", "CsAfrPwrConfig", "PwrConfig"]: + code += "#[derive(Default)]\n" + item(hw, "pub(crate) struct " + name + " {") + "\n" + code += item(hw, "impl PState {") + "\n" + code += "}\nmod initdata { use crate::{prelude::*,hw}; const PSTATE_TABLE_ENTRIES: usize = 16;\n" + code += item(init, "pub(crate) struct PStateTables {") + "\n" + code += item(init, "impl PStateTables {") + "\n}\n" + code += "mod m3_mmio_26 { pub const SLOTS:usize=53; pub const RECORD_BYTES:usize=40; }\n" + code += f'#[path = "{M3}/m3_power_26.rs"] mod m3_power_26;\n' + code += fixture + for path in [M3 / "hw.rs", M3 / "initdata.rs", M3 / "m3_power_26.rs", ROOT / "drivers/gpu/drm/asahi/float.rs"]: + print(hashlib.sha256(path.read_bytes()).hexdigest(), path.relative_to(ROOT), flush=True) + with tempfile.TemporaryDirectory(prefix="m3-power-") as directory: + source = Path(directory) / "test.rs" + source.write_text(code) + for flags in [[], ["-O"]]: + binary = Path(directory) / "tests" + subprocess.run(["rustc", "--edition=2021", "--test", *flags, str(source), "-o", str(binary)], check=True) + subprocess.run([str(binary)], check=True) + + +if __name__ == "__main__": + main() From ee69dab429ed9f6498c03303dd34c9eb4af44ff2 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:40:41 -0400 Subject: [PATCH 20/37] drm/asahi: prepare extended firmware queues and controller handoff Construct native queue, pool, context and notifier records. Read controller inputs from the admitted OPP handoff, preserve nanosecond durations and reject unsupported optional flags. Prepare native startup without enabling extended admission. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_adt_config.rs | 57 +++++++- drivers/gpu/drm/asahi/m3/m3_compute_layout.rs | 21 +++ .../gpu/drm/asahi/m3/m3_compute_sequence.rs | 110 +++++++++++++++ .../gpu/drm/asahi/m3/m3_compute_storage.rs | 66 +++++++++ drivers/gpu/drm/asahi/m3/m3_config.rs | 79 ++++++++++- drivers/gpu/drm/asahi/m3/m3_device.rs | 6 + drivers/gpu/drm/asahi/m3/m3_drm.rs | 3 +- drivers/gpu/drm/asahi/m3/m3_pool.rs | 31 +++++ drivers/gpu/drm/asahi/m3/m3_pool_layout.rs | 128 ++++++++++++++++++ drivers/gpu/drm/asahi/m3/m3_profile.rs | 20 +++ drivers/gpu/drm/asahi/m3/m3_queue_layout.rs | 41 ++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 57 ++++++++ drivers/gpu/drm/asahi/m3/m3_state_layout.rs | 52 +++++++ drivers/gpu/drm/asahi/m3/m3_submit.rs | 8 +- drivers/gpu/drm/asahi/mmu.rs | 4 +- tools/asahi/m3-runtime-tests.rs | 10 ++ 16 files changed, 678 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs index 1893c113cf2a7..355cd44b04025 100644 --- a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs @@ -260,7 +260,14 @@ impl PstatePolicy { /// The runtime backend's InitData contents: the generated object images and the /// performance-state policy. +pub(crate) struct ExtendedInputs { + pub(crate) power: hw::PwrConfig, + pub(crate) sram_uv: KVec, + pub(crate) bases: super::m3_mmio_26::RegisterBases, +} + pub(crate) struct Contents { + pub(crate) extended: Option, /// Generated images, per owner; None for owners that stay zero. Only the three image /// owners' images are uploaded; the records' are checked. images: KVec>>, @@ -304,6 +311,10 @@ impl Contents { experiment: Option<&Experiment>, ) -> Result { let dev = pdev.as_ref(); + if firmware.profile() == super::m3_profile::Profile::Extended { + if experiment.is_some() { return Err(EINVAL); } + return Self::extended(dev, soc); + } let param = m3_params::initdata_param(); check_version(dev, firmware, experiment)?; let iomaps = match experiment { @@ -370,12 +381,54 @@ impl Contents { ); let read_only_slots = experiment.map(|e| e.read_only_slots()).unwrap_or(0); let hwdata_object = soc.hwdata_object; - Ok(Contents { images, pstates, iomaps, table, read_only_slots, hwdata_object }) + Ok(Contents { extended: None, images, pstates, iomaps, table, read_only_slots, hwdata_object }) + } + + fn extended(dev: &device::Device, soc: &Soc) -> Result { + let cfg = soc.hwcfg.ok_or(ENODEV)?; + if soc.chip_id != 0x8122 || cfg.chip_id != 0x8122 { return Err(ENODEV); } + let power = hw::PwrConfig::load(dev, cfg)?; + let tables = initdata::PStateTables::new(&power, true, None, false)?; + if tables.len != 9 || tables.max != 8 || tables.max_frequency_khz(&power) != 1_338_000 { + return Err(EINVAL); + } + let node = dev.of_node().ok_or(ENODEV)?; + let opps = node.parse_phandle(c_str!("operating-points-v2"), 0).ok_or(EINVAL)?; + let mut sram_uv = KVec::new(); + for opp in opps.children() { + let values: KVec = opp.get_property(c_str!("opp-microvolt-sram"))?; + if values.len() != 1 { return Err(EINVAL); } + sram_uv.push(values[0], GFP_KERNEL)?; + } + if sram_uv.len() != power.perf_states.len() { return Err(EINVAL); } + let block = |index: usize| -> Result { + Ok(cfg.io_mappings.get(index).and_then(|value| value.as_ref()).ok_or(EINVAL)?.base as u64) + }; + let bases = super::m3_mmio_26::RegisterBases { + gpu: soc.windows.sgx, interrupt: block(2)?, + memory: block(11)?.checked_add(0x1c4000).ok_or(EOVERFLOW)?, + telemetry_write: block(18)?, telemetry_read: block(19)?, + neural: block(25)?, metrology: block(26)?, + }; + let mut freqs = [0; 16]; + for (index, value) in freqs.iter_mut().enumerate().take(tables.len) { + *value = tables.primary_state(&power, index).freq_hz / 1_000_000; + } + // This profile enables the native firmware's thermal/PID controllers. + // The legacy host-only thermal governor must not replace those inputs. + let pstates = PstatePolicy { table_max: 8, table_max_mhz: 1338, + max: 8, safe: 8, boot: 1, max_mhz: 1338, safe_mhz: 1338, + boot_mhz: freqs[1], freqs, thermal: ThermalMode::Off }; + let mut images = KVec::new(); + for _ in 0..super::m3_init_26::COUNT { images.push(None, GFP_KERNEL)?; } + Ok(Self { extended: Some(ExtendedInputs { power, sram_uv, bases }), + images, pstates, iomaps: &[], table: (8, 1338), read_only_slots: 0, hwdata_object: None }) } /// Check the contents before they are uploaded. pub(crate) fn check_for_upload(&self) -> Result { - if self.images.len() == storage::COUNT { + let count = if self.extended.is_some() { super::m3_init_26::COUNT } else { storage::COUNT }; + if self.images.len() == count { Ok(()) } else { Err(EINVAL) diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs b/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs index 64c8cb005c4a1..ace58e38f38a6 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute_layout.rs @@ -71,7 +71,14 @@ impl Command { /// Firmware envelopes vary independently of the GPU's CDM stream format. pub(crate) fn encode_for(&self, profile: super::m3_profile::Profile, asid_generation: u8, event_slot: u8, out: &mut [u8]) -> Result<(), Error> { + self.encode_with_usc(profile, asid_generation, event_slot, 0, out) + } + + pub(crate) fn encode_with_usc(&self, profile: super::m3_profile::Profile, + asid_generation: u8, event_slot: u8, usc_base: u64, out: &mut [u8]) -> Result<(), Error> { let extended = profile == super::m3_profile::Profile::Extended; + if usc_base & 0xffff_ffff != 0 || usc_base >= 1 << 42 + || (!extended && usc_base != 0) { return Err(Error::Address); } let size = if extended { EXTENDED_SIZE } else { SIZE }; if out.len() < size { return Err(Error::Size); } if extended && (self.register_set != RegisterSet::G15G || event_slot >= 4 @@ -122,6 +129,7 @@ impl Command { emit(0x1a540, 1); emit(0x014a9, 1); emit(0x0a351, 1); + if usc_base != 0 { emit(0x10071, usc_base); } } pointer(bytes, 0x720, registers.get()); bytes[0x728..0x72a].copy_from_slice(&(count as u16).to_le_bytes()); @@ -232,4 +240,17 @@ mod tests { assert_eq!(read64(&b, META), c.stamp.get()); assert_eq!(&b[SIZE..], &[0xa5; EXTENDED_SIZE - SIZE]); } + #[test] + fn optional_usc_window_is_a_dispatch_register_without_stream_rewriting() { + let c = fixture(); + let mut output = [0xa5; EXTENDED_SIZE]; + c.encode_with_usc(Profile::Extended, 1, 0, 1 << 32, &mut output).unwrap(); + assert_eq!(read32(&output, 0x728) & 0xffff, 25); + assert_eq!(read32(&output, REGISTERS + 24 * REGISTER_STRIDE), 0x10071); + assert_eq!(read64(&output, REGISTERS + 24 * REGISTER_STRIDE + 4), 1 << 32); + output.fill(0xa5); + assert!(c.encode_with_usc(Profile::Extended, 1, 0, 1, &mut output).is_err()); + assert_eq!(output, [0xa5; EXTENDED_SIZE]); + } + } diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs b/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs index a10c3040d570f..bd78c03823aa0 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs @@ -82,3 +82,113 @@ impl Sequence { Ok(()) } } + +impl Sequence { + pub(crate) const EXTENDED_LENGTH: usize = 0x300; + pub(crate) const EXTENDED_ATTACHMENTS: usize = 0x58; + pub(crate) const EXTENDED_HAS_ATTACHMENTS: usize = 0x2c0; + + /// The extended firmware operates on a native G15 CDM stream. Its firmware + /// microsequence has wider timestamp operations and different engine state. + pub(crate) fn encode_extended(&self, auxiliary: command::GpuVa, + context_store: super::m3_pool_layout::GpuRegion, queue_sequence: u64, + output: &mut [u8]) -> Result<(), Error> { + if self.context >= 64 || self.generation != 0 { return Err(Error::Address); } + if context_store.size() < 0x20000 { return Err(Error::Size); } + let registers = self.command.at(command::REGISTERS, 24 * command::REGISTER_STRIDE)?; + let engine = self.command.at(0x76c, 0x60)?; + let clock = self.command.at(0x82c, 8)?; + let pairs = self.command.at(0x834, 16)?; + let end_pair = self.command.at(0x83c, 8)?; + let pool_pointer = self.command.at(0x86a, 8)?; + let status = self.command.at(0x8ac, 4)?; + let time_aux = self.command.at(0x89c, 8)?; + let elapsed = self.command.at(0x7dc, 8)?; + let tail = self.command.at(0x88c, 8)?; + let activity = self.stats.at(0, 8)?; + let bytes = output.get_mut(..STORAGE).ok_or(Error::Size)?; + bytes.fill(0); + word(bytes, 0, 0x0b); + pointer(bytes, 0x14, registers); + pointer(bytes, 0x1c, activity); + pointer(bytes, 0x24, self.queue.get()); + word(bytes, 0x2c, self.context); + word(bytes, 0x30, 1); + pointer(bytes, 0x38, queue_sequence); + pointer(bytes, 0x44, engine); + word(bytes, 0x50, 1); + pointer(bytes, 0x160, self.pool.get()); + pointer(bytes, 0x178, auxiliary.get()); + for (offset, delta) in [0, 0xf400, 0x1e800, 0x1f000].into_iter().enumerate() { + pointer(bytes, 0x188 + offset * 8, context_store.base() + delta); + } + pointer(bytes, 0x1a8, status); + word(bytes, 0x1b8, 2); + // Timestamp records are 76 bytes; WFI sits between the two records. + for (offset, opcode, selected) in [(0x1bc, 0x8000_0003, pairs), (0x20c, 3, end_pair)] { + word(bytes, offset, opcode); + pointer(bytes, offset + 4, clock); + pointer(bytes, offset + 0xc, pairs); + pointer(bytes, offset + 0x14, selected); + pointer(bytes, offset + 0x1c, self.queue.get()); + pointer(bytes, offset + 0x2c, time_aux); + pointer(bytes, offset + 0x34, elapsed); + pointer(bytes, offset + 0x44, 0xc10000); + } + word(bytes, 0x208, 1); + let finish = 0x258; + word(bytes, finish, 0x0c); + pointer(bytes, finish + 4, activity); + pointer(bytes, finish + 0xc, self.queue.get()); + word(bytes, finish + 0x14, self.context); + pointer(bytes, finish + 0x18, engine); + pointer(bytes, finish + 0x24, 0xc10000); + pointer(bytes, finish + 0x2c, self.fw_stamp.get()); + word(bytes, finish + 0x34, self.stamp_value); + pointer(bytes, finish + 0x5c, pool_pointer); + word(bytes, finish + 0x64, (-(finish as i32)) as u32); + pointer(bytes, finish + 0x69, status); + pointer(bytes, finish + 0x71, tail); + word(bytes, 0x2d4, 0x4000_0002); + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + fn fixture() -> Sequence { + let fw = |i: u64| FirmwareVa::new(0xffff_fc21_a000_0000 + i * 0x4000).unwrap(); + Sequence { command: Region::new(fw(0).get(), command::EXTENDED_SIZE).unwrap(), + stats: Region::new(fw(1).get(), 0xe10).unwrap(), + notifier: Region::new(fw(2).get(), 0x200).unwrap(), queue: fw(3), + context: 3, generation: 0, pool: fw(4), scratch: [fw(5); 5], + fw_stamp: fw(6), stamp_value: 0x700 } + } + fn ptr(bytes: &[u8], at: usize) -> u64 { u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()) } + #[test] fn extended_timestamps_and_retirement_refer_to_their_live_owners() { + let s = fixture(); + let mut output = [0xa5; STORAGE + 8]; + s.encode_extended(command::GpuVa::new(0x5000_0000).unwrap(), + super::super::m3_pool_layout::GpuRegion::new(0x6000_0000, 0x20000).unwrap(), 7, &mut output).unwrap(); + assert_eq!(ptr(&output, 0x1c), s.stats.at(0, 8).unwrap()); + assert_eq!(ptr(&output, 0x38), 7); + assert_eq!(ptr(&output, 0x188), 0x6000_0000); + assert_eq!(ptr(&output, 0x1bc + 0x14), s.command.at(0x834, 8).unwrap()); + assert_eq!(ptr(&output, 0x20c + 0x14), s.command.at(0x83c, 8).unwrap()); + assert_eq!(ptr(&output, 0x258 + 0x2c), s.fw_stamp.get()); + assert_eq!(ptr(&output, 0x258 + 0x5c), s.command.at(0x86a, 8).unwrap()); + assert_eq!(&output[STORAGE..], &[0xa5; 8]); + } + #[test] fn extended_sequence_rejects_legacy_command_extents_and_event_generations() { + for generation in [0, 1] { + let mut s = fixture(); + s.generation = generation; + if generation == 0 { s.command = Region::new(s.command.at(0, 1).unwrap(), command::SIZE).unwrap(); } + let mut output = [0xa5; STORAGE]; + assert!(s.encode_extended(command::GpuVa::new(0x5000_0000).unwrap(), + super::super::m3_pool_layout::GpuRegion::new(0x6000_0000, 0x20000).unwrap(), 0, &mut output).is_err()); + assert_eq!(output, [0xa5; STORAGE]); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs b/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs index 6bd3aecda387d..a897a45f0922d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute_storage.rs @@ -87,3 +87,69 @@ pub(crate) fn allocation(index:usize)->Result { pub(crate) fn coalesce_stamp_flush(wide:bool,timestamps:[u64;2],last:bool)->bool { wide && timestamps==[0,0] && !last } + +pub(crate) fn allocation_for(profile: super::m3_profile::Profile, index: usize) + -> Result { + let mut owner = allocation(index)?; + if profile == super::m3_profile::Profile::Legacy { return Ok(owner); } + match index { + // The initial allocator has eight 4-MiB payload owners. Retain the + // historical indices without allocating unused 4-MiB payloads. + 9..=22 => { owner.size = 0x4000; owner.mapping = Mapping::FirmwareOnly; } + POOL_PAGES => owner.size = 0x20000, + POOL_TABLE => { owner.size = 0x4000; owner.mapping = Mapping::Client(POOL_TABLE_VA & !0x3fff); } + POOL => owner.size = 0xa0, + GPU_CONTEXT => owner.size = 0x38, + NOTIFIER => owner.size = 0x200, + EVENT => owner.size = 8, + QUEUE => owner.size = 0x24c0, + PREEMPTION => owner.size = 0x4000, + COMMAND => { owner.size = 0x8b0; owner.mapping = Mapping::CommandAlias(0x73_0000_0000); } + TAIL_SCRATCH => { owner.size = 0x4000; owner.mapping = Mapping::Client(0x72_0100_0000); } + 46 => { owner.size = 0x40000; owner.mapping = Mapping::Client(0x72_0000_0000); } + 45 | 47..=49 => { owner.size = 0x4000; owner.mapping = Mapping::FirmwareOnly; } + _ => {} + } + Ok(owner) +} + +pub(crate) fn slot_offset_for(profile: super::m3_profile::Profile, index: usize, slot: usize) + -> Result { + if slot >= SLOTS { return Err(InvalidOwner); } + let size = allocation_for(profile, index)?.size; + Ok(if RESET.contains(&index) { ((size + 0x3fff) & !0x3fff) * slot } else { 0 }) +} + +pub(crate) fn batch_allocation_for(profile: super::m3_profile::Profile, index: usize) + -> Result { + let mut owner = allocation_for(profile, index)?; + owner.size += slot_offset_for(profile, index, SLOTS - 1)?; + Ok(owner) +} + +#[cfg(test)] +mod extended_tests { + use super::*; + use super::super::m3_profile::Profile; + #[test] fn extended_commands_do_not_straddle_another_slots_page() { + let command = allocation_for(Profile::Extended, COMMAND).unwrap(); + let Mapping::CommandAlias(base) = command.mapping else { panic!("missing command alias") }; + assert_eq!(base & 0x3fff, 0); + for slot in 0..SLOTS - 1 { + let start = slot_offset_for(Profile::Extended, COMMAND, slot).unwrap(); + let next = slot_offset_for(Profile::Extended, COMMAND, slot + 1).unwrap(); + assert!(start + command.size <= next); + } + assert_eq!(batch_allocation_for(Profile::Extended, QUEUE).unwrap().size, 0x24c0); + assert!(slot_offset_for(Profile::Extended, COMMAND, SLOTS).is_err()); + } + #[test] fn legacy_plan_is_byte_for_byte_the_same_for_every_owner_and_slot() { + for index in 0..COUNT { + assert_eq!(allocation_for(Profile::Legacy, index), allocation(index)); + for slot in 0..SLOTS { + assert_eq!(slot_offset_for(Profile::Legacy, index, slot), slot_offset(index, slot)); + } + assert_eq!(batch_allocation_for(Profile::Legacy, index), batch_allocation(index)); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index 67f4f963637f2..bfb408b07a530 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -51,6 +51,7 @@ pub(crate) struct Config { objects: KVec, profile: super::m3_profile::Profile, control_generation: u32, + io_descriptors: Option>, _iomaps: KVec, pub(crate) completed_events: u64, pstates: crate::m3::m3_adt_config::PstateWatch, @@ -67,6 +68,17 @@ impl Config { pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, firmware: &crate::m3::m3_firmware::Firmware, contents: &crate::m3::m3_adt_config::Contents) -> Result { contents.check_for_upload()?; + if let Some(inputs) = &contents.extended { + if firmware.profile() != super::m3_profile::Profile::Extended { return Err(EINVAL); } + let owners = ExtendedStorage::new(dev, uat, &inputs.power, &inputs.sram_uv, inputs.bases)?; + let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?; + return Ok(Self { objects: owners.objects, _iomaps: owners.iomaps, + profile: firmware.profile(), control_generation: 0, completed_events: 0, + io_descriptors: Some(owners.descriptors), + pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal, + request_top: 8, request_value: 0, request_checked: false, request_check_at: None, + request_writes: 0, request_observed: 0 }); + } let mut objects = KVec::new(); let mut iomaps = KVec::new(); for index in 0..storage::COUNT { @@ -142,7 +154,7 @@ impl Config { crate::m3::m3_adt_config::check_upload(dev.as_ref().as_ref(), firmware, contents.iomaps, &mut objects)?; agx_memory::publish(); let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?; - Ok(Self { objects, profile: firmware.profile(), control_generation: 0, + Ok(Self { objects, profile: firmware.profile(), control_generation: 0, io_descriptors: None, _iomaps: iomaps, completed_events: 0, pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal, request_top:contents.table.0,request_value:0,request_checked:false, @@ -156,6 +168,15 @@ impl Config { objects[RUNTIME_POINTERS].write(0, &records.runtime)?; objects[CONTROL_REGION].write(0, &records.control) } + pub(crate) fn publish_io_table(&self, + pdev: &kernel::platform::Device>, + firmware: &crate::m3::m3_firmware::Firmware) -> Result { + if let Some(descriptors) = &self.io_descriptors { + firmware.publish_io_table(pdev, descriptors)?; + } + Ok(()) + } + pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDrmDevice)->Result { let value=self.objects[GLOBALS].read_u32(crate::m3::m3_globals_debug::OFFSET)?; @@ -164,6 +185,8 @@ impl Config { Ok(()) } pub(crate) fn performance_request(&mut self,chip:u32,busy:bool)->Result { + // Native controller inputs supersede the legacy fixed-request interface. + if self.profile == super::m3_profile::Profile::Extended { return Ok(()); } let Some(value)=crate::m3::m3_consistent_perf::request(chip, crate::m3::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; if value==0 && self.request_value==0 {return Ok(());} @@ -195,6 +218,14 @@ impl Config { /// state the firmware reports against the ceiling and the runtime cap. Called after boot and /// after every job, by the job thread. pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDrmDevice, device: &crate::m3::m3_device::Device, what: &str) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + let state = device.pstate_register()? & 15; + if state > self.request_top { + dev_err!(dev.as_ref(), "M3: native performance state {} exceeds {} ({})\n", state, self.request_top, what); + return Err(ERANGE); + } + return Ok(()); + } let now = kernel::time::Instant::::now(); if let Some(cap) = self.thermal.update(dev.as_ref().as_ref(), now) { crate::m3::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; @@ -206,13 +237,25 @@ impl Config { Ok(()) } pub(crate) fn stats_region(&self) -> Result { - let owner = &self.objects[HARDWARE_DATA]; + let owner = &self.objects[if self.profile == super::m3_profile::Profile::Extended { + UNKNOWN_C3 + } else { HARDWARE_DATA }]; init::Region::new(owner.va(), owner.size()).map_err(|_| EINVAL) } pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDrmDevice, device: &crate::m3::m3_device::Device, sequence: u32) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + let state = device.pstate_register()? & 15; + let performance = &mut self.objects[super::m3_init_26::PERFORMANCE_CONTROL]; + let actual = performance.read_u32(0x2c)?; + let target = performance.read_u32(0x30)?; + let hardware = performance.read_u32(0x38)?; + dev_info!(dev.as_ref(), "M3_NATIVE_POWER sample={} state={} actual={} target={} hardware={}\n", + sequence, state, actual, target, hardware); + return Ok(()); + } let cap = self.thermal.allowed(kernel::time::Instant::::now()); self.pstates.log_active(dev.as_ref().as_ref(), &mut self.objects[HARDWARE_DATA], device.pstate_register()?, cap, sequence)?; @@ -232,6 +275,7 @@ impl Config { } pub(crate) fn render_pb(&mut self) -> Result { + if self.profile == super::m3_profile::Profile::Extended { return Err(EOPNOTSUPP); } self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3::m3_render::PB_PAGES).map_err(|_|EINVAL)?) } pub(crate) fn submit_queue(&mut self, kind:usize, queue:u64, head:u16,event:u8,new:bool)->Result { @@ -274,6 +318,13 @@ impl Config { && self.objects[RUNTIME_FLAGS].read_u32(4)? == 1) } pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDrmDevice) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + let ready = self.objects[RUNTIME_FLAGS].read_u32(0x14)?; + let consumed = self.objects[CONTROL_REGION].read_u32(0xeae8)?; + dev_info!(dev.as_ref(), "M3: extended firmware ready={} control-generation={}/{}\n", + ready, consumed, self.control_generation); + return Ok(()); + } let magic=self.objects[RUNTIME_POINTERS].read_u32(0x314)?; let state=self.objects[RUNTIME_POINTERS].read_u32(0x2fc)?; let a=self.objects[RUNTIME_FLAGS].read_u32(4)?; @@ -284,12 +335,14 @@ impl Config { } /// The firmware's halt counter and its halted and resume flags. pub(crate) fn halt_state(&mut self) -> Result<(u64,u32,u32)> { + if self.profile == super::m3_profile::Profile::Extended { return Err(EOPNOTSUPP); } let control=&mut self.objects[CONTROL_REGION]; Ok((control.read_u64(0x4580)?,control.read_u32(0x4590)?,control.read_u32(0x45a0)?)) } /// Let a halted firmware continue: clear halted, then set resume. These /// words keep the halt/resume layout of earlier Apple GPU firmware. pub(crate) fn resume_halted(&mut self) -> Result { + if self.profile == super::m3_profile::Profile::Extended { return Err(EOPNOTSUPP); } let control=&mut self.objects[CONTROL_REGION]; control.u32(0x4590,0)?; agx_memory::publish(); @@ -303,6 +356,18 @@ impl Config { /// at +0x40a0, and the details it leaves for a recovery event at +0x44a0. /// Nothing is acknowledged, cleared or resumed here. pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDrmDevice, events: u64) -> Result { + if self.profile == super::m3_profile::Profile::Extended { + for (owner, start, length) in [(super::m3_init_26::RECOVERY_ACTIVITY, 0, 0x60), + (CONTROL_REGION, 0x4ebc, 0x100)] { + for offset in (start..start + length).step_by(32) { + let mut bytes = [0; 32]; + self.objects[owner].read(offset, &mut bytes)?; + dev_err!(dev.as_ref(), "M3: firmware recovery owner={} offset={:#x} bytes={:02x?}\n", + owner, offset, bytes); + } + } + return Ok(()); + } let control=&mut self.objects[CONTROL_REGION]; let halt_count=control.read_u64(0x4580)?; let halted=control.read_u32(0x4590)?; @@ -318,7 +383,7 @@ impl Config { /// index (at most 16), as `drain` reads them. Nothing is consumed or acknowledged, and the /// completion count is not changed. pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDrmDevice) -> Result { - const SIZE: usize = 0x38; + let size = if self.profile == super::m3_profile::Profile::Extended { 0x48 } else { 0x38 }; const COUNT: usize = 256; let r = self.objects[storage::EVENT*2].read_u32(0)? as usize; let w = self.objects[storage::EVENT*2].read_u32(0x20)? as usize; @@ -327,8 +392,8 @@ impl Config { let mut index = r; for _ in 0..16 { if index == w { break; } - let mut raw = [0u8; SIZE]; - self.objects[storage::EVENT*2+1].read(index * SIZE, &mut raw)?; + let mut raw = [0u8; 0x48]; + self.objects[storage::EVENT*2+1].read(index * size, &mut raw[..size])?; dev_info!(dev.as_ref(), "M3 event ring entry {} kind={} bytes={:02x?}\n", index, u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]), raw); index = (index + 1) % COUNT; @@ -357,7 +422,9 @@ impl Config { let state = storage::DEVICE_CONTROL * 2; let cursors = [self.objects[state].read_u32(0)?, self.objects[state].read_u32(0x10)?, self.objects[state].read_u32(0x20)?]; - let cursor = format::idle_cursor(cursors).map_err(|_| EBUSY)?; + let cursor = format::idle_cursor(cursors).map_err(|error| { + if error == init::Error::Bounds { EIO } else { EBUSY } + })?; let consumed = self.objects[CONTROL_REGION].read_u32(0xeae8)?; let generation = action.generation(self.control_generation, consumed).map_err(|_| EOVERFLOW)?; let mut record = [0; format::CONTROL_BYTES]; diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index c43017a8933c0..05a3961369a6d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -176,6 +176,12 @@ impl Device { /// Read-only snapshot. Do not select or acknowledge fault banks here; /// the firmware owns those registers until recovery is ported. pub(crate) fn log_engine_state(&self, vm: &crate::mmu::Vm) -> Result { + if self.firmware.profile() == super::m3_profile::Profile::Extended { + // Adaptive clock gating can leave individual engine banks inaccessible + // even while the GPU performance-state register says powered up. + dev_info!(self.dev.as_ref(), "M3: adaptive firmware fault evidence uses retained metadata; engine banks may be gated\n"); + return Ok(()); + } let sgx_guard = self.sgx.try_access().ok_or(ENODEV)?; let sgx = sgx_guard.as_view(); let pstate = sgx.try_read32(0xe01000)? & 0xf; diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 796abc7793f90..e1db1adf9e6a9 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -258,7 +258,8 @@ impl gpu::Gpu for Backend { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64) -> Result> { Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, - self.soc.features.fragment_dependency)?,GFP_KERNEL)?) + self.feature_flags() & uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY as u64 != 0, + self.feature_flags() & uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY as u64 != 0)?,GFP_KERNEL)?) } fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range) diff --git a/drivers/gpu/drm/asahi/m3/m3_pool.rs b/drivers/gpu/drm/asahi/m3/m3_pool.rs index ab50ab7eca8a1..4eb284e1291bd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pool.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pool.rs @@ -39,3 +39,34 @@ impl Owners { objects[self.manager].write(0, &manager) } } + +impl Owners { + pub(crate) fn initialize_extended(&self, objects: &mut [Buffer], slot: u8, + compute_only: bool) -> Result { + let mut backings = KVec::new(); + for index in self.backing.clone().take(8) { + let buffer = objects.get(index).ok_or(EINVAL)?; + backings.push(wire::GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?, GFP_KERNEL)?; + } + let backing = wire::ModernBacking(&backings); + let populated = backing.pages().map_err(|_| EINVAL)?; + let pages = objects.get(self.pages).ok_or(EINVAL)?; + let entries = wire::GpuRegion::new(pages.gpu_va()?, pages.size()).map_err(|_| EINVAL)?; + let table = objects.get(self.table).ok_or(EINVAL)?; + let groups = wire::GpuRegion::new(table.gpu_va()?, table.size()).map_err(|_| EINVAL)?; + if groups.size() < populated / 512 * 64 { return Err(EINVAL); } + let descriptor = wire::ModernPool { entries, groups, + submitted: FirmwareVa::new(objects.get(self.counter).ok_or(EINVAL)?.va()).map_err(|_| EINVAL)?, + identity: FirmwareVa::new(objects.get(self.manager).ok_or(EINVAL)?.va()).map_err(|_| EINVAL)?, + slot, capacity: (entries.size() / 8).try_into()?, populated: populated.try_into()?, compute_only }; + let mut bytes = [0; wire::ModernPool::SIZE]; + descriptor.encode(&mut bytes).map_err(|_| EINVAL)?; + for index in 0..populated { + objects[self.pages].u64(index * 8, backing.page(index).map_err(|_| EINVAL)?)?; + } + for index in 0..populated / 512 { + objects[self.table].write(index * 64, &backing.group(index).map_err(|_| EINVAL)?)?; + } + objects[self.manager].write(0, &bytes) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs b/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs index a8474d262bc26..ac7e6771f7880 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pool_layout.rs @@ -69,3 +69,131 @@ impl<'a> Pool<'a> { Ok(()) } } + +/// Extended firmware's private-memory pool descriptor. The owners and initial +/// population are supplied separately from the descriptor's firmware state. +pub(crate) struct ModernPool { + pub(crate) identity: FirmwareVa, + pub(crate) slot: u8, + pub(crate) entries: GpuRegion, + pub(crate) groups: GpuRegion, + pub(crate) submitted: FirmwareVa, + pub(crate) capacity: u32, + pub(crate) populated: u32, + pub(crate) compute_only: bool, +} + +impl ModernPool { + pub(crate) const SIZE: usize = 0xa0; + + pub(crate) fn encode(&self, output: &mut [u8]) -> Result<(), Error> { + let entry_bytes = (self.capacity as usize).checked_mul(8).ok_or(Error::Size)?; + let group_bytes = (self.populated as usize / 512).checked_mul(8).ok_or(Error::Size)?; + if !(1..=64).contains(&self.slot) || self.capacity == 0 || self.populated >= self.capacity || self.populated % 512 != 0 + || self.entries.base() & 0x3fff != 0 || self.groups.base() & 0x3fff != 0 + || self.entries.size() < entry_bytes || self.groups.size() < group_bytes + || self.submitted.get() & 7 != 0 || self.identity.get() & 7 != 0 { return Err(Error::Address); } + let group_count: u32 = group_bytes.try_into().map_err(|_| Error::Size)?; + let bytes = output.get_mut(..Self::SIZE).ok_or(Error::Size)?; + bytes.fill(0); + let word = |bytes: &mut [u8], offset: usize, value: u32| { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + }; + bytes[..8].copy_from_slice(&self.identity.get().to_le_bytes()); + word(bytes, 8, self.slot.into()); + word(bytes, 12, (self.populated == 0).into()); + word(bytes, 16, 2); + bytes[0x14..0x1c].copy_from_slice(&self.entries.base().to_le_bytes()); + word(bytes, 0x1c, self.capacity); + word(bytes, 0x24, self.populated); + word(bytes, 0x28, (self.populated == 0).into()); + word(bytes, 0x2c, self.populated); + bytes[0x30..0x38].copy_from_slice(&self.groups.base().to_le_bytes()); + word(bytes, 0x40, 4); + word(bytes, 0x48, group_count); + bytes[0x4c..0x54].copy_from_slice(&self.submitted.get().to_le_bytes()); + word(bytes, 0x5c, (self.compute_only && self.populated != 0).into()); + word(bytes, 0x84, (self.populated != 0).into()); + word(bytes, 0x94, 2); + Ok(()) + } +} + +#[cfg(test)] +mod modern_tests { + use super::*; + fn pool(compute_only: bool) -> ModernPool { + ModernPool { identity: FirmwareVa::new(0xffff_fc21_9100_0000).unwrap(), slot: 1, + entries: GpuRegion::new(0x5000_0000, 0x20000).unwrap(), + groups: GpuRegion::new(0x6000_0000, 0x4000).unwrap(), + submitted: FirmwareVa::new(0xffff_fc21_9000_0000).unwrap(), + capacity: 0x4000, populated: 512, compute_only } + } + fn word(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) + } + #[test] fn shared_render_pool_never_gets_compute_exclusion() { + for compute in [false, true] { + let mut output = [0xa5; ModernPool::SIZE + 8]; + pool(compute).encode(&mut output).unwrap(); + assert_eq!(word(&output, 0x5c), compute.into()); + assert_eq!(word(&output, 0x48), 8); + assert_eq!(word(&output, 0x84), 1); + assert_eq!(&output[ModernPool::SIZE..], &[0xa5; 8]); + } + } + #[test] fn bad_population_and_insufficient_owners_reject_before_mutation() { + for pages in [1, 511, 513, 0x4000] { + let mut value = pool(true); value.populated = pages; + let mut output = [0xa5; ModernPool::SIZE]; + assert!(value.encode(&mut output).is_err()); + assert_eq!(output, [0xa5; ModernPool::SIZE]); + } + let mut value = pool(true); value.capacity = 0x4001; + let mut output = [0xa5; ModernPool::SIZE]; + assert!(value.encode(&mut output).is_err()); + assert_eq!(output, [0xa5; ModernPool::SIZE]); + } +} + +/// Populate the extended freelist from independently retained 4-MiB owners. +/// Hardware group records describe 2-MiB subregions, not entire owners. +pub(crate) struct ModernBacking<'a>(pub(crate) &'a [GpuRegion]); +impl ModernBacking<'_> { + pub(crate) fn pages(&self) -> Result { + if self.0.is_empty() || self.0.len() > 8 + || self.0.iter().any(|owner| owner.size() != BACKING_SIZE || owner.base() & 0x3fff != 0) { + return Err(Error::Size); + } + Ok(self.0.len() * BACKING_SIZE / PAGE_SIZE) + } + pub(crate) fn page(&self, index: usize) -> Result { + if index >= self.pages()? { return Err(Error::Index); } + let per_owner = BACKING_SIZE / PAGE_SIZE; + Ok(self.0[index / per_owner].base() + (index % per_owner * PAGE_SIZE) as u64) + } + pub(crate) fn group(&self, index: usize) -> Result<[u8; 64], Error> { + if index >= self.pages()? / 512 { return Err(Error::Index); } + let base = self.page(index * 512)?; + let mut record = [0; 64]; + record[..8].copy_from_slice(&(base | (1 << 61)).to_le_bytes()); + Ok(record) + } +} + +#[cfg(test)] +mod backing_tests { + use super::*; + #[test] fn group_records_never_cross_a_noncontiguous_backing_owner() { + let owners = [GpuRegion::new(0x5000_0000, BACKING_SIZE).unwrap(), + GpuRegion::new(0x6000_0000, BACKING_SIZE).unwrap()]; + let backing = ModernBacking(&owners); + assert_eq!(backing.pages(), Ok(2048)); + assert_eq!(backing.page(1024), Ok(0x6000_0000)); + let second = backing.group(2).unwrap(); + assert_eq!(u64::from_le_bytes(second[..8].try_into().unwrap()), (1 << 61) | 0x6000_0000); + assert_eq!(&second[8..], &[0; 56]); + assert!(backing.group(4).is_err()); + assert!(backing.page(2048).is_err()); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_profile.rs b/drivers/gpu/drm/asahi/m3/m3_profile.rs index c70a88029072b..004ac2c7d1eef 100644 --- a/drivers/gpu/drm/asahi/m3/m3_profile.rs +++ b/drivers/gpu/drm/asahi/m3/m3_profile.rs @@ -32,6 +32,13 @@ impl Profile { } } + pub(crate) fn gpu_duration_ns(self, elapsed: u64) -> Option { + match self { + Self::Legacy => elapsed.checked_mul(1000).map(|value| value / 24), + Self::Extended => Some(elapsed), + } + } + pub(crate) const fn vm_acknowledged(self, params_bytes: u64, flags: u32) -> bool { // Both profiles consume G15 records; the existing VM padding stays zero. params_bytes >= PARAMS_REQUIRED && flags == 0 @@ -66,4 +73,17 @@ mod tests { assert_eq!(Profile::Extended.user_end(), 0x3ff_ffff_8000); assert_eq!(Profile::Legacy.user_end(), 0x2ff_ffff_8000); } + #[test] + fn native_nanoseconds_are_never_rescaled_as_legacy_ticks() { + assert_eq!( + Profile::Extended.gpu_duration_ns(24_000_000), + Some(24_000_000) + ); + assert_eq!( + Profile::Legacy.gpu_duration_ns(24_000_000), + Some(1_000_000_000) + ); + assert_eq!(Profile::Extended.gpu_duration_ns(u64::MAX), Some(u64::MAX)); + assert_eq!(Profile::Legacy.gpu_duration_ns(u64::MAX), None); + } } diff --git a/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs b/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs index 3daba7acba2d0..8fd5ffdf1d30d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_queue_layout.rs @@ -113,3 +113,44 @@ pub(crate) fn ring_entry(index: u32, capacity: u32, seeds: &[FirmwareVa]) -> Res if seeds.is_empty() || seeds.len() > capacity as usize || capacity > 65536 || index >= capacity { return Err(Error::Index); } Ok(if (index as usize) < seeds.len() { seeds[index as usize].0.to_le_bytes() } else { [0xcc; 8] }) } + +impl QueueInfo { + pub(crate) const EXTENDED_SIZE: usize = 0x24c0; + + /// Scheduler storage belongs to this same retained allocation; it is not + /// interchangeable with the legacy GPU scratch object. + pub(crate) fn encode_extended(&self, owner: FirmwareVa, output: &mut [u8]) -> Result<(), Error> { + let end = owner.get().checked_add(Self::EXTENDED_SIZE as u64).ok_or(Error::Address)?; + if end > 0xffff_fc30_0000_0000 { return Err(Error::Address); } + let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + for (offset, value) in [(0, self.state.get()), (8, self.ring.get()), + (16, self.job_list.get()), (24, owner.get() + 0xb0), (0xa4, self.context.get())] { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + for offset in [0x2c, 0x4c] { bytes[offset..offset + 4].fill(0xff); } + for offset in [0x30, 0x34, 0x48] { + bytes[offset..offset + 4].copy_from_slice(&2u32.to_le_bytes()); + } + bytes[0x38..0x40].copy_from_slice(&0xffff_0000_0000_0000u64.to_le_bytes()); + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn scheduler_pointer_relocates_with_its_retained_queue_owner() { + let fw = |i: u64| FirmwareVa::new(0xffff_fc21_4000_0000 + i * 0x4000).unwrap(); + let info = QueueInfo { state: fw(1), ring: fw(2), job_list: fw(3), + gpu_scratch: fw(4), context: fw(5), uuid: 9 }; + let mut output = vec![0xa5; QueueInfo::EXTENDED_SIZE + 8]; + info.encode_extended(fw(0), &mut output).unwrap(); + assert_eq!(u64::from_le_bytes(output[24..32].try_into().unwrap()), fw(0).get() + 0xb0); + assert_eq!(&output[0xb0..QueueInfo::EXTENDED_SIZE], &vec![0; QueueInfo::EXTENDED_SIZE - 0xb0]); + assert_eq!(&output[QueueInfo::EXTENDED_SIZE..], &[0xa5; 8]); + let mut short = [0xa5; INFO_SIZE]; + assert!(info.encode_extended(fw(0), &mut short).is_err()); + assert_eq!(short, [0xa5; INFO_SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 1a366d7f4b340..40399a80a01f3 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -1214,6 +1214,9 @@ impl Runtime { } fn boot_inner(&mut self, pdev: &platform::Device>) -> Result { use crate::m3::t8122_start::BootStep; + if self.inner.device.firmware().profile() == super::m3_profile::Profile::Extended { + return self.boot_extended(pdev); + } let root = self.inner.config.root(); dev_info!(pdev.as_ref(), "M3: publishing owned initdata {:#x}\n", root); self.inner.boot_step = BootStep::Publish; @@ -1245,6 +1248,60 @@ impl Runtime { dev_info!(pdev.as_ref(), "M3: firmware accepted owned initdata and device controls\n"); Ok(()) } + fn boot_extended(&mut self, pdev: &platform::Device>) -> Result { + use super::m3_channel_format::Action; + use crate::m3::t8122_start::BootStep; + // The first control is pending when the root becomes visible. Its + // retirement comes from InitData consumption, without a work kick. + let first = self.inner.config.control_extended(Action::Initialize)?; + self.inner.config.publish_io_table(pdev, self.inner.device.firmware())?; + let root = self.inner.config.root(); + Pin::new(&mut self.inner.transport).send_message(0x20, + 0x0081_0000_0000_0000 | (root & ((1u64 << 44) - 1)))?; + self.inner.boot_step = BootStep::AwaitReady; + let deadline = Instant::::now(); + while !self.inner.config.ready()? { + let inner: &mut Inner = &mut *self.inner; + inner.config.drain(&inner.drm)?; + if !inner.state.healthy() { return Err(EIO); } + if deadline.elapsed() >= Delta::from_secs(2) { return Err(ETIMEDOUT); } + fsleep(Delta::from_millis(1)); + } + self.retire_extended_control(first)?; + self.inner.boot_step = BootStep::DeviceControl; + // Each data master registers its two firmware engine classes. + for ordinal in 0..6 { + let action = Action::Register { pipe: ordinal / 2, engine: ordinal % 2 + 1 }; + self.send_extended_control(action)?; + } + self.inner.boot_step = BootStep::PostReady; + self.send_extended_control(Action::Idle { enabled: true })?; + let inner: &mut Inner = &mut *self.inner; + inner.config.check_pstate(&inner.drm, &inner.device, "after native boot")?; + inner.config.log_ready(&inner.drm)?; + inner.state.health.set_gpu_pending(false); + dev_info!(pdev.as_ref(), "M3: extended firmware ready; native controllers and idle policy admitted\n"); + Ok(()) + } + + fn send_extended_control(&mut self, action: super::m3_channel_format::Action) -> Result { + let cursor = self.inner.config.control_extended(action)?; + Pin::new(&mut self.inner.transport).send_message(0x21, 0x0084_0000_0000_0011)?; + self.retire_extended_control(cursor) + } + + fn retire_extended_control(&mut self, expected: u32) -> Result { + let deadline = Instant::::now(); + loop { + let inner: &mut Inner = &mut *self.inner; + inner.config.drain(&inner.drm)?; + if !inner.state.healthy() { return Err(EIO); } + if inner.config.control_done(expected)? { return Ok(()); } + if deadline.elapsed() >= Delta::from_secs(2) { return Err(ETIMEDOUT); } + fsleep(Delta::from_millis(1)); + } + } + fn send_control(&mut self, opcode: u32) -> Result { let next = match self.inner.config.control(opcode) { Ok(next) => next, diff --git a/drivers/gpu/drm/asahi/m3/m3_state_layout.rs b/drivers/gpu/drm/asahi/m3/m3_state_layout.rs index 199b7082b26c2..6eeafea666e74 100644 --- a/drivers/gpu/drm/asahi/m3/m3_state_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_state_layout.rs @@ -66,3 +66,55 @@ pub(crate) fn manager_misc(out: &mut [u8]) -> Result<(), Error> { word(bytes, 0x20, 1); // cpu_flag / reset Ok(()) } + +pub(crate) const EXTENDED_CONTEXT_SIZE: usize = 0x38; +pub(crate) const EXTENDED_NOTIFIER_SIZE: usize = 0x200; + +pub(crate) fn context_extended(output: &mut [u8]) -> Result<(), Error> { + let bytes = output.get_mut(..EXTENDED_CONTEXT_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + bytes[..2].fill(0xff); + bytes[5] = 1; + bytes[0x26] = 2; + bytes[0x33] = 0xff; + Ok(()) +} + +impl Notifier { + pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> { + if self.context >= 64 { return Err(Error::Address); } + let bytes = output.get_mut(..EXTENDED_NOTIFIER_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + bytes[..8].copy_from_slice(&self.threshold.get().to_le_bytes()); + word(bytes, 0x10, 0x50); + word(bytes, NOTIFIER_CONTEXT, self.context); + bytes[0xa8..0xb0].fill(0xff); + word(bytes, 0x114, 1); + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn registration_state_is_left_to_firmware() { + let mut bytes = [0xa5; EXTENDED_CONTEXT_SIZE + 8]; + context_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[..6], &[255, 255, 0, 0, 0, 1]); + assert_eq!(bytes[0x26], 2); + assert_eq!(bytes[0x33], 255); + assert_eq!(&bytes[EXTENDED_CONTEXT_SIZE..], &[0xa5; 8]); + } + #[test] fn notifier_has_zero_generation_and_distinct_firmware_storage() { + let notifier = Notifier { threshold: FirmwareVa::new(0xffff_fc21_7000_0000).unwrap(), context: 3 }; + let mut bytes = [0xa5; EXTENDED_NOTIFIER_SIZE + 8]; + notifier.encode_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[8..12], &[0; 4]); + assert_eq!(&bytes[0xa8..0xb0], &[255; 8]); + assert_eq!(&bytes[0x114..0x118], &[1, 0, 0, 0]); + assert_eq!(&bytes[EXTENDED_NOTIFIER_SIZE..], &[0xa5; 8]); + let mut short = [0xa5; NOTIFIER_SIZE]; + assert!(notifier.encode_extended(&mut short).is_err()); + assert_eq!(short, [0xa5; NOTIFIER_SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index ffc57c77afcb2..2aac0dfad36ff 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -176,11 +176,12 @@ pub(crate) struct Queue { shared:Shared,vm:mmu::Vm,usc:u64,fences:FenceContexts, /// Whether the SoC advertises early tiling (`m3_soc::Features::fragment_dependency`). early_tiling:bool, + wide_visibility:bool, urgency:u8, } impl Queue { pub(crate) fn new(shared:Shared,scheduler:Arc>,vm:mmu::Vm,priority:u32,usc:u64, - early_tiling:bool)->Result { + early_tiling:bool, wide_visibility:bool)->Result { agx_uapi::QueueUscWindow{base:usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}.validate().map_err(|_|EINVAL)?; // file.rs passes REALTIME - UAPI priority: 3 is UAPI LOW, 0 is UAPI REALTIME. let urgency=3u8.saturating_sub(priority.min(3) as u8); @@ -188,7 +189,7 @@ impl Queue { 1=>sched::Priority::High,0=>sched::Priority::Kernel,_=>return Err(EINVAL)}; Ok(Self {entity:sched::Entity::new(&scheduler,priority)?,_scheduler:scheduler, shared,vm,usc,fences:FenceContexts::new(1,c_str!("asahi_m3_queue"),kernel::static_lock_class!())?, - early_tiling,urgency}) + early_tiling,wide_visibility,urgency}) } } impl queue::Queue for Queue { @@ -216,7 +217,8 @@ impl queue::Queue for Queue { let agx_uapi::ParsedHardwareCommand::Compute{mut payload,attachments,..}=command else {return Err(ENOTSUPP);}; // Other engines and helpers need their own M3 firmware translation. let wide=kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_WIDE_VISIBILITY as u32; - if payload.flags & !wide !=0 || payload.sampler_count!=0 || payload.sampler_heap!=0 { + if payload.flags & !wide !=0 || (!self.wide_visibility && payload.flags & wide != 0) + || payload.sampler_count!=0 || payload.sampler_heap!=0 { pr_err!("M3 compute rejected: flags={:#x} samplers={} heap={:#x} attachments={} timestamps={}/{}\n",payload.flags,payload.sampler_count,payload.sampler_heap,attachments.as_slice().len(),payload.timestamps.start.handle,payload.timestamps.end.handle); return Err(ENOTSUPP); } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 437685bfad1de..2651d0aca3198 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1155,8 +1155,7 @@ impl VmBind { self.1 } - /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. - #[cfg(CONFIG_DRM_ASAHI_G16)] + /// Stable firmware wire incarnation for this active root lease; zero is unbound. pub(crate) fn generation(&self) -> u8 { self.0.binding.lock().generation } @@ -2703,7 +2702,6 @@ impl Vm { } /// Spend the binding generation only for an incarnation exposed to firmware. - #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn mark_binding_published(&self) -> Result { let uat = self.gpuvm.lock_inner().uat_inner.clone(); let binding = self.binding.lock(); diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index abac06668c0a1..458603366c954 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -36,3 +36,13 @@ mod m3_compute_layout; #[path = "../../drivers/gpu/drm/asahi/m3/m3_channel_format.rs"] mod m3_channel_format; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_sequence.rs"] +mod m3_compute_sequence; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_state_layout.rs"] +mod m3_state_layout; + + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_storage.rs"] +mod m3_compute_storage; From c5e1b35de06e10f3587132f93ea51bac621c6510 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:40:41 -0400 Subject: [PATCH 21/37] drm/asahi: dispatch G15 compute through extended T8122 firmware Dispatch extended firmware compute records through the existing M3 runtime while preserving G15 CDM streams. Retain owned pool backing and leave firmware-managed register setup to the extended interface. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_compute.rs | 162 +++++++++++++++++-------- drivers/gpu/drm/asahi/m3/m3_config.rs | 10 ++ drivers/gpu/drm/asahi/m3/m3_device.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 18 ++- 4 files changed, 132 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index 34aa0c0d2cd3f..577510b7e9ecf 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -40,25 +40,28 @@ pub(crate) struct Compute { barriers: Option, /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). registers: command::RegisterSet, + profile: super::m3_profile::Profile, /// Last: drops after the command aliases. _lower_tlbi: crate::m3::m3_memory::LowerTlbi, } impl Compute { pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, stats: Region, control: crate::m3::agx_compute::Control, - registers: command::RegisterSet) -> Result { + registers: command::RegisterSet, profile: super::m3_profile::Profile) -> Result { let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new(); let mut client_command=None; let mut command_bytes = KVec::new(); - command_bytes.resize(command::SIZE, 0u8, GFP_KERNEL)?; + command_bytes.resize(if profile == super::m3_profile::Profile::Extended { + command::EXTENDED_SIZE + } else { command::SIZE }, 0u8, GFP_KERNEL)?; let mut sequence_bytes = KVec::new(); sequence_bytes.resize(microsequence::STORAGE, 0u8, GFP_KERNEL)?; // A private compute aperture, disjoint from the retained render graph. // Address-bearing graph fields are constructed from their live owners. for index in 0..storage::COUNT { - let a=storage::batch_allocation(index).map_err(|_|EINVAL)?; + let a=storage::batch_allocation_for(profile,index).map_err(|_|EINVAL)?; let mut b=match a.mapping { Mapping::CommandAlias(va)=>Buffer::at(dev,uat.kernel_vm(),Some((uat.kernel_lower_vm(),va)),None,a.size,true)?, Mapping::Client(va)=>Buffer::at(dev,uat.kernel_vm(),Some((vm,va)),None,a.size,true)?, @@ -68,10 +71,13 @@ impl Compute { if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);} objects.push(b,GFP_KERNEL)?; } - Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, registers)?; - Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0)?; - UMA_POOL.initialize(&mut objects)?; - Self::initialize_state(&mut objects, binding.slot())?; + Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, + registers, profile, binding.generation())?; + Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0, profile, 0)?; + if profile == super::m3_profile::Profile::Extended { + UMA_POOL.initialize_extended(&mut objects, 1, true)?; + } else { UMA_POOL.initialize(&mut objects)?; } + Self::initialize_state(&mut objects, binding.slot(), profile)?; // Queue references are firmware VAs from live allocations. In contrast // to render's unpublished seed entries, compute dispatches entry zero // immediately, so its initial producer cursor remains one. @@ -80,8 +86,13 @@ impl Compute { job_list: address(JOB_LIST)?, gpu_scratch: address(QUEUE_SCRATCH)?, context: address(GPU_CONTEXT)?, uuid: 0xc10000 }; let command = address(storage::COMMAND)?; - let mut bytes = [0; queue::INFO_SIZE]; - info.encode(&mut bytes).map_err(|_| EINVAL)?; + let mut bytes = KVec::new(); + bytes.resize(if profile == super::m3_profile::Profile::Extended { + queue::QueueInfo::EXTENDED_SIZE + } else { queue::INFO_SIZE }, 0u8, GFP_KERNEL)?; + if profile == super::m3_profile::Profile::Extended { + info.encode_extended(address(storage::QUEUE)?, &mut bytes).map_err(|_| EINVAL)?; + } else { info.encode(&mut bytes).map_err(|_| EINVAL)?; } objects[storage::QUEUE].write(0, &bytes)?; let mut state = [0; queue::STATE_SIZE]; queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 1 } @@ -107,54 +118,74 @@ impl Compute { // the invalidation. crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; - Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,_lower_tlbi:lower_tlbi}) + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,profile,_lower_tlbi:lower_tlbi}) } fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, control: crate::m3::agx_compute::Control, sequence: u64, slot: usize, - register_set: command::RegisterSet) -> Result { - let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + register_set: command::RegisterSet, profile: super::m3_profile::Profile, generation: u8) -> Result { + let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); let preemption = &objects[PREEMPTION]; let value = command::Command { context, counter: timeline::previous(sequence), notifier: fw(NOTIFIER)?, - preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation(PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, + preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation_for(profile,PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, cdm: command::GpuVa::new(control.base).map_err(|_| EINVAL)?, cdm_last: command::GpuVa::new(control.end.checked_sub(4).ok_or(EINVAL)?).map_err(|_| EINVAL)?, gpu_alias: command::GpuVa::new(objects[storage::COMMAND].gpu_va()?+offset(storage::COMMAND)? as u64).map_err(|_| EINVAL)?, - microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: microsequence::LENGTH as u32, + microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: + if profile == super::m3_profile::Profile::Extended { microsequence::Sequence::EXTENDED_LENGTH as u32 } + else { microsequence::LENGTH as u32 }, stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?, stamp_value: timeline::stamp(0xc1000000,sequence), timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?], pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false, register_set, }; - value.encode(bytes).map_err(|_| EINVAL)?; + value.encode_with_usc(profile, generation, 0, + if profile == super::m3_profile::Profile::Extended { control.usc_base } else { 0 }, bytes).map_err(|_| EINVAL)?; objects[storage::COMMAND].write(offset(storage::COMMAND)?, bytes) } fn encode_sequence(objects: &mut [Buffer], bytes: &mut [u8], stats: Region, - context: u32, stamp_value: u32, slot: usize) -> Result { - let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + context: u32, stamp_value: u32, slot: usize, profile: super::m3_profile::Profile, + queue_sequence: u64) -> Result { + let offset = |owner:usize| storage::slot_offset_for(profile,owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); - let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation(owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); + let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation_for(profile,owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); let value = microsequence::Sequence { command: region(storage::COMMAND)?, stats, notifier: region(NOTIFIER)?, - queue: fw(storage::QUEUE)?, context, generation: 1, pool: fw(storage::POOL)?, + queue: fw(storage::QUEUE)?, context, + generation: if profile == super::m3_profile::Profile::Extended { 0 } else { 1 }, pool: fw(storage::POOL)?, scratch: [fw(MICROSEQUENCE_SCRATCH[0])?, fw(MICROSEQUENCE_SCRATCH[1])?, fw(MICROSEQUENCE_SCRATCH[2])?, fw(MICROSEQUENCE_SCRATCH[3])?, fw(MICROSEQUENCE_SCRATCH[4])?], fw_stamp: fw(storage::FW_STAMP)?, stamp_value, }; - value.encode(bytes).map_err(|_| EINVAL)?; + if profile == super::m3_profile::Profile::Extended { + let auxiliary = command::GpuVa::new(objects[TAIL_SCRATCH].gpu_va()? + offset(TAIL_SCRATCH)? as u64).map_err(|_| EINVAL)?; + let context_store = GpuRegion::new(objects[MICROSEQUENCE_SCRATCH[1]].gpu_va()? + + offset(MICROSEQUENCE_SCRATCH[1])? as u64, 0x40000).map_err(|_| EINVAL)?; + value.encode_extended(auxiliary, context_store, queue_sequence, bytes).map_err(|_| EINVAL)?; + } else { value.encode(bytes).map_err(|_| EINVAL)?; } objects[storage::MICROSEQUENCE].write(offset(storage::MICROSEQUENCE)?, bytes) } - fn initialize_state(objects: &mut [Buffer], context_slot: u32) -> Result { + fn initialize_state(objects: &mut [Buffer], context_slot: u32, profile: super::m3_profile::Profile) -> Result { let address = |owner: usize| queue::FirmwareVa::new(objects[owner].va()).map_err(|_| EINVAL); let notification = state::Notifier { threshold: address(storage::EVENT)?, context: context_slot }; let list_owner = address(JOB_LIST)?; let mut context = [0; state::CONTEXT_SIZE]; - state::context(&mut context).map_err(|_| EINVAL)?; - objects[GPU_CONTEXT].write(0, &context)?; - let mut notifier = [0; state::NOTIFIER_SIZE]; - notification.encode(&mut notifier).map_err(|_| EINVAL)?; + if profile == super::m3_profile::Profile::Extended { + state::context_extended(&mut context).map_err(|_| EINVAL)?; + } else { state::context(&mut context).map_err(|_| EINVAL)?; } + let context_size = if profile == super::m3_profile::Profile::Extended { + state::EXTENDED_CONTEXT_SIZE + } else { state::CONTEXT_SIZE }; + objects[GPU_CONTEXT].write(0, &context[..context_size])?; + let mut notifier = KVec::new(); + notifier.resize(if profile == super::m3_profile::Profile::Extended { + state::EXTENDED_NOTIFIER_SIZE + } else { state::NOTIFIER_SIZE }, 0u8, GFP_KERNEL)?; + if profile == super::m3_profile::Profile::Extended { + notification.encode_extended(&mut notifier).map_err(|_| EINVAL)?; + } else { notification.encode(&mut notifier).map_err(|_| EINVAL)?; } objects[NOTIFIER].write(0, ¬ifier)?; let mut list = [0; state::JOB_LIST_SIZE]; state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; @@ -182,7 +213,7 @@ impl Compute { let binding = uat.bind(vm)?; let mut views = KVec::new(); for i in 0..storage::COUNT { - if matches!(storage::allocation(i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { + if matches!(storage::allocation_for(self.profile,i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { views.push((i,self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; } } @@ -221,18 +252,17 @@ impl Compute { if self.batch_count>=storage::SLOTS {return Err(E2BIG);} self.slot=self.batch_count; for i in storage::RESET { - let offset=storage::slot_offset(i,self.slot).map_err(|_|EINVAL)?; - let size=storage::allocation(i).map_err(|_|EINVAL)?.size; + let offset=storage::slot_offset_for(self.profile,i,self.slot).map_err(|_|EINVAL)?; + let size=storage::allocation_for(self.profile,i).map_err(|_|EINVAL)?.size; self.objects[i].fill_range(offset,size,if i==PREEMPTION {0xcc} else {0})?; } self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?; self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?; - Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers)?; + Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers,self.profile,self._binding.generation())?; let stamp=self.stamp(); - Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot)?; - self.objects[storage::EVENT].u32(0,self.sequence as u32)?; - self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; - let address=self.objects[storage::COMMAND].va()+storage::slot_offset(storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; + Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot,self.profile,self.sequence - 1)?; + self.write_counters()?; + let address=self.objects[storage::COMMAND].va()+storage::slot_offset_for(self.profile,storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; self.objects[storage::RING].u64(usize::from(self.head)*8,address)?; self.head=(self.head+1)%queue::QUALIFIED_CAPACITY as u16; self.batch_count+=1; @@ -245,39 +275,55 @@ impl Compute { pub(crate) fn abort_batch(&mut self)->Result { (self.sequence,self.head,self.batch_count)=self.checkpoint; self.slot=self.batch_count.saturating_sub(1); - self.objects[storage::EVENT].u32(0,self.sequence as u32)?; - self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; + self.write_counters()?; self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; agx_memory::publish();Ok(()) } + fn write_counters(&mut self) -> Result { + for index in [storage::EVENT, storage::COUNTER] { + if self.profile == super::m3_profile::Profile::Extended { + self.objects[index].u64(0, self.sequence)?; + } else { self.objects[index].u32(0, self.sequence as u32)?; } + } + Ok(()) + } + fn offset(&self,owner:usize)->Result { - storage::slot_offset(owner,self.slot).map_err(|_|EINVAL) + storage::slot_offset_for(self.profile,owner,self.slot).map_err(|_|EINVAL) } /// Same cache-generation contract as M4: StartCompute and the persistent /// notifier must use the same tag before the firmware flushes user stamps. /// RTKit 2419: StartCompute.counter1 +0x34; JobMeta.flush_stamps +0x804. pub(crate) fn prepare_completion(&mut self,coalesce:bool)->Result { + if coalesce && self.profile == super::m3_profile::Profile::Extended { return Err(EINVAL); } let sequence=self.offset(storage::MICROSEQUENCE)?; let command=self.offset(storage::COMMAND)?; self.objects[storage::MICROSEQUENCE].u32(sequence+microsequence::GENERATION,0)?; - self.objects[storage::COMMAND].u32(command+command::FLUSH_STAMPS,u32::from(!coalesce))?; + self.objects[storage::COMMAND].u32(command+if self.profile == super::m3_profile::Profile::Extended { 0x81c } else { command::FLUSH_STAMPS },u32::from(!coalesce))?; agx_memory::publish();Ok(()) } pub(crate) fn set_user_timestamps(&mut self,addresses:[u64;2])->Result { - let offset=self.offset(storage::COMMAND)?+command::USER_TIMESTAMPS; + let offset=self.offset(storage::COMMAND)?+if self.profile == super::m3_profile::Profile::Extended { + 0x844 + } else { command::USER_TIMESTAMPS }; self.objects[storage::COMMAND].u64(offset,addresses[0])?; self.objects[storage::COMMAND].u64(offset+8,addresses[1])?; let pointer=if addresses==[0,0] {0} else {self.objects[storage::COMMAND].va()+offset as u64}; let sequence=self.offset(storage::MICROSEQUENCE)?; - for start in [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] { + for start in if self.profile == super::m3_profile::Profile::Extended { [0x1bc, 0x20c] } + else { [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] } { self.objects[storage::MICROSEQUENCE].u64(sequence+start+microsequence::USER_TIMESTAMP_POINTER,pointer)?; } agx_memory::publish();Ok(()) } pub(crate) fn set_attachments(&mut self,attachments:&crate::m3::agx_attachments::Attachments)->Result { let sequence=self.offset(storage::MICROSEQUENCE)?; - self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::ATTACHMENTS,&attachments.0)?; - self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::HAS_ATTACHMENTS,&[u8::from(attachments.count()!=0)])?; + self.objects[storage::MICROSEQUENCE].write(sequence+if self.profile == super::m3_profile::Profile::Extended { + microsequence::Sequence::EXTENDED_ATTACHMENTS + } else { microsequence::ATTACHMENTS },&attachments.0)?; + self.objects[storage::MICROSEQUENCE].write(sequence+if self.profile == super::m3_profile::Profile::Extended { + microsequence::Sequence::EXTENDED_HAS_ATTACHMENTS + } else { microsequence::HAS_ATTACHMENTS },&[u8::from(attachments.count()!=0)])?; agx_memory::publish();Ok(()) } fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } @@ -319,26 +365,31 @@ impl Compute { agx_memory::publish();crate::mem::sync();Ok(()) } pub(crate) fn gpu_ns(&mut self)->Result { - let mut total=0; - for slot in 0..self.batch_count {total+=self.slot_gpu_ns(slot)?;} + let mut total: u64 = 0; + for slot in 0..self.batch_count { + total = total.checked_add(self.slot_gpu_ns(slot)?).ok_or(EOVERFLOW)?; + } Ok(total) } pub(crate) fn slot_gpu_ns(&mut self,slot:usize)->Result { if slot>=self.batch_count {return Err(EINVAL);} - let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; - let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; - Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; + let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; + if start == 0 || end < start { return Ok(0); } + self.profile.gpu_duration_ns(end - start).ok_or(EOVERFLOW) } /// A batch span includes gaps between its compute commands, unlike gpu_ns. /// Read only after stamps, queues, events and idle state prove retirement. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, - self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) + Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, + self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) } pub(crate) fn head(&self) -> u16 { self.head } pub(crate) fn ordinal(&self)->u64 {self.sequence} - pub(crate) fn first(&self) -> bool { self.sequence == self.batch_count as u64 } + pub(crate) fn first(&self) -> bool { + self.profile == super::m3_profile::Profile::Legacy && self.sequence == self.batch_count as u64 + } pub(crate) fn progress(&mut self, dev: &driver::AsahiDrmDevice) -> Result { // Report real completions by elapsed time as well as sequence count; // batches need not land on a multiple of 128. @@ -361,10 +412,15 @@ impl Compute { pub(crate) fn queue(&self)->u64 {self.objects[storage::QUEUE].va()} pub(crate) fn complete(&mut self)->Result { - Ok(self.objects[storage::STAMP].read_u32(0)?==self.stamp() - && self.objects[storage::FW_STAMP].read_u32(0)?==self.stamp() - && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?==u32::from(self.head) - && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?==u32::from(self.head)) + let stamped = self.objects[storage::STAMP].read_u32(0)? == self.stamp() + && self.objects[storage::FW_STAMP].read_u32(0)? == self.stamp() + && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)? == u32::from(self.head) + && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)? == u32::from(self.head); + if !stamped { return Ok(false); } + if self.profile == super::m3_profile::Profile::Extended { + return Ok(self.objects[storage::POOL].read_u64(0x54)? == self.sequence); + } + Ok(true) } pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { for off in (0..512).step_by(128) { diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index bfb408b07a530..ffd1bb963b905 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -177,6 +177,16 @@ impl Config { Ok(()) } + pub(crate) fn global_table_views(&mut self, vm: &mmu::Vm) -> Result> { + let mut mappings = KVec::new(); + if self.profile == super::m3_profile::Profile::Extended { + for index in [super::m3_init_26::GLOBAL_POOL, super::m3_init_26::GLOBAL_PARAMETERS] { + mappings.push(self.objects[index].map_gpu_view(vm)?, GFP_KERNEL)?; + } + } + Ok(mappings) + } + pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDrmDevice)->Result { let value=self.objects[GLOBALS].read_u32(crate::m3::m3_globals_debug::OFFSET)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index 05a3961369a6d..474a8b3f561ec 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -129,6 +129,7 @@ impl Device { } e.sgx_setup() } + None if device.firmware.profile() == super::m3_profile::Profile::Extended => None, None => Some(soc.sgx_setup.ok_or(ENODEV)?), }; if let Some((setup_offset, setup_value)) = setup { @@ -179,7 +180,6 @@ impl Device { if self.firmware.profile() == super::m3_profile::Profile::Extended { // Adaptive clock gating can leave individual engine banks inaccessible // even while the GPU performance-state register says powered up. - dev_info!(self.dev.as_ref(), "M3: adaptive firmware fault evidence uses retained metadata; engine banks may be gated\n"); return Ok(()); } let sgx_guard = self.sgx.try_access().ok_or(ENODEV)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 40399a80a01f3..058c6610399ae 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -389,7 +389,7 @@ impl Inner { let index=found.unwrap_or(self.jobs.len()); if found.is_none() { let job=match control { - crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), }; self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; @@ -469,6 +469,9 @@ impl Inner { crate::mem::sync(); self.t8122_verdict=false; let previous_events=self.config.completed_events; + if self.device.firmware().profile() == super::m3_profile::Profile::Extended { + packet.vm.mark_binding_published().map_err(early)?; + } let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; self.state.health.set_gpu_pending(true); self.gpu_pending = true; @@ -535,7 +538,7 @@ impl Inner { let control=first_compute.ok_or(EINVAL)?; if existing.is_none() { let job=crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, - self.config.stats_region()?,control,self.device.soc().registers)?; + self.config.stats_region()?,control,self.device.soc().registers,self.device.firmware().profile())?; self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?; } else { let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; @@ -1059,7 +1062,7 @@ impl Runtime { // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as // every context's upper half, and the host window (root entry 2) through the // shared L2. The RTKit is awake and no initdata or GPU job has been published. - let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources)?); + let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources,device.firmware().profile())?); let config=Config::new(&drm,uat,device.firmware(),&contents)?; // Link every kernel table the configuration created before the firmware sees an // address in it, then drop any translation cached before they existed. @@ -1103,7 +1106,8 @@ impl Runtime { // (7.1.12's M3 VM job lifetime, the shared `mmu::lifetime` on 7.2). let vm=self.inner.uat.new_vm(id, range)?.with_tracked_status()?; let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?; - vm.install_driver_mappings(KVec::new(),reserved)?; + let mappings = self.inner.config.global_table_views(&vm)?; + vm.install_driver_mappings(mappings,reserved)?; Ok(vm) } pub(crate) fn map_timestamp(&mut self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { @@ -1369,13 +1373,15 @@ pub(crate) fn physical_counter() -> u64 { /// (ttbs, pagetables, handoff, shared-l2) and the PPL exchange with its startup epoch, as the /// G15 backend does it on 7.2 (`g15::uat`). fn adopt_uat(drm: &driver::AsahiDrmDevice, soc: &'static crate::m3::m3_soc::Soc, - resources: &crate::m3::agx_resources::Resources) -> Result<(mmu::Uat, crate::g15::uat::Window)> { + resources: &crate::m3::agx_resources::Resources, profile: super::m3_profile::Profile) + -> Result<(mmu::Uat, crate::g15::uat::Window)> { let cfg = soc.hwcfg.ok_or(ENODEV)?; let [ttbs, tables, handoff, shared_l2, ..] = resources.regions; let range = |r: crate::m3::agx_resources::Region| r.base..r.base + r.size; let geometry = mmu::UatGeometry { ias: cfg.uat_ias.into(), oas: cfg.uat_oas }; let regions = mmu::AdoptedRegions { handoff: range(handoff), ttbs: range(ttbs), kernel_root: tables.base }; - let uat = mmu::Uat::new_adopted(drm, geometry, ®ions)?; + let uat = mmu::Uat::new_adopted_with_user_upper(drm, geometry, ®ions, + profile == super::m3_profile::Profile::Legacy)?; let mut window = crate::g15::uat::Window::from_regions(handoff.base, ttbs.base, tables.base, shared_l2.base)?; window.bootstrap(drm.as_ref().as_ref(), &uat)?; // 7.1.12 cleaned every reserved-table write to the point of coherency; these regions may be From ec30b227fefa70ee9bb6b962bceb2bf03495d791 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Fri, 30 Oct 2026 21:40:42 -0400 Subject: [PATCH 22/37] drm/asahi: enable J613 extended firmware render and retirement Encode extended tiler and fragment work, scene records, independent notifications and attachment visibility through the existing runtime. Retire native pools and enable only the exact J613 26.6.2 profile. Keep the public UAPI and physical G15 stream formats unchanged. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_board.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_compute.rs | 10 +- drivers/gpu/drm/asahi/m3/m3_config.rs | 12 + drivers/gpu/drm/asahi/m3/m3_drm.rs | 10 +- drivers/gpu/drm/asahi/m3/m3_firmware.rs | 10 +- .../gpu/drm/asahi/m3/m3_fragment_command.rs | 138 ++++++++++-- .../gpu/drm/asahi/m3/m3_parameter_layout.rs | 74 +++++++ drivers/gpu/drm/asahi/m3/m3_pass.rs | 4 +- drivers/gpu/drm/asahi/m3/m3_pass_layout.rs | 36 +++ drivers/gpu/drm/asahi/m3/m3_profile.rs | 21 ++ drivers/gpu/drm/asahi/m3/m3_render.rs | 206 +++++++++++++----- .../gpu/drm/asahi/m3/m3_render_sequence.rs | 168 ++++++++++++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 4 +- drivers/gpu/drm/asahi/m3/m3_scene_layout.rs | 40 ++++ drivers/gpu/drm/asahi/m3/m3_shared_layout.rs | 26 +++ drivers/gpu/drm/asahi/m3/m3_submit.rs | 7 +- drivers/gpu/drm/asahi/m3/m3_sync_layout.rs | 55 +++++ drivers/gpu/drm/asahi/m3/m3_tiler_command.rs | 131 +++++++++-- drivers/gpu/drm/asahi/m3/t8122_admission.rs | 40 +++- tools/asahi/m3-runtime-tests.rs | 60 +++++ 20 files changed, 953 insertions(+), 101 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs index c44b2a42d99d5..5d4ea4277b618 100644 --- a/drivers/gpu/drm/asahi/m3/m3_board.rs +++ b/drivers/gpu/drm/asahi/m3/m3_board.rs @@ -13,7 +13,7 @@ use crate::m3::m3_resources::{ use crate::m3::m3_soc::Soc; /// Whether the root node's compatible list contains `compatible`. -fn board_is(compatible: &[u8]) -> bool { +pub(crate) fn board_is(compatible: &[u8]) -> bool { let Some(root) = kernel::of::root() else { return false; }; diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index 577510b7e9ecf..b6e9444ed5449 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -350,10 +350,12 @@ impl Compute { let barriers=self.barriers.as_mut().ok_or(EINVAL)?; let offset=self.slot*64; let mut bytes=[0;crate::m3::m3_sync_layout::BARRIER_SIZE]; - crate::m3::m3_sync_layout::Barrier { + let barrier=crate::m3::m3_sync_layout::Barrier { stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event, self_value:value,uuid:0xc10000,dependency:crate::m3::m3_sync_layout::Dependency::PreviousPass, - }.encode(&mut bytes).map_err(|_|EINVAL)?; + }; + if self.profile==super::m3_profile::Profile::Extended {barrier.encode_extended(&mut bytes)} + else {barrier.encode(&mut bytes)}.map_err(|_|EINVAL)?; barriers.write(offset,&bytes)?; let address=barriers.va()+offset as u64; let (slots,next)=queue::insert_before_last(self.head,queue::QUALIFIED_CAPACITY).map_err(|_|EINVAL)?; @@ -382,8 +384,8 @@ impl Compute { /// Read only after stamps, queues, events and idle state prove retirement. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, - self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) + Ok([self.profile.profiler_ticks(self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?), + self.profile.profiler_ticks(self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset_for(self.profile,TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?)]) } pub(crate) fn head(&self) -> u16 { self.head } pub(crate) fn ordinal(&self)->u64 {self.sequence} diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index ffd1bb963b905..02b847e210dc9 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -246,6 +246,18 @@ impl Config { self.thermal.busy(now); Ok(()) } + pub(crate) fn render_stats(&self) -> Result<[init::Region;2]> { + if self.profile==super::m3_profile::Profile::Legacy { + let value=self.stats_region()?;return Ok([value;2]); + } + let region=|index:usize,offset:usize| { + let object=&self.objects[index]; + init::Region::new(object.va().checked_add(offset as u64).ok_or(EOVERFLOW)?, + object.size().checked_sub(offset).ok_or(EINVAL)?).map_err(|_|EINVAL) + }; + Ok([region(UNKNOWN_C0,4)?,region(UNKNOWN_C1,8)?]) + } + pub(crate) fn stats_region(&self) -> Result { let owner = &self.objects[if self.profile == super::m3_profile::Profile::Extended { UNKNOWN_C3 diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index e1db1adf9e6a9..8aedd920f31dc 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -113,10 +113,12 @@ impl Backend { crate::m3::t8122_start::not_admitted(pdev.as_ref(), soc); })?; dev_info!(pdev.as_ref(), "M3: resource admission complete\n"); - // An armed T8122 start supplies the values its table leaves out. An unarmed T8122 - // needs a complete table, which it does not have: it is refused here. - let experiment = crate::m3::t8122_start::arm(pdev.as_ref(), soc)?; - if experiment.is_none() { + // Extended T8122 uses its dedicated initialization graph. Keep the + // legacy guarded-start and completeness checks for older interfaces. + let extended=crate::m3::t8122_start::is_t8122(soc) + && crate::m3::t8122_admission::loaded_profile(pdev)?==super::m3_profile::Profile::Extended; + let experiment = if extended {None} else {crate::m3::t8122_start::arm(pdev.as_ref(), soc)?}; + if experiment.is_none() && !extended { soc.require_complete(pdev.as_ref())?; } let experiment = experiment.as_ref(); diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index 6bc87a33aaa05..94a2d3384d08a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -62,7 +62,7 @@ pub(crate) static KNOWN_IMAGES_T8122: [KnownImage; 1] = [KnownImage { initdata_magic: Some(G15_V14_8_3_INITDATA), }]; -/// Image record for the extended format. The SoC's runtime selection does not use it yet. +/// Exact normalized image identity for the extended firmware interface. pub(crate) static T8122_IMAGES_26: [KnownImage; 1] = [KnownImage { name: "T8122 C0 firmware-compat 26.6.2", uuid: None, @@ -164,7 +164,7 @@ pub(crate) static T8122_LAYOUT: Layout = Layout { pub(crate) const TEXT_SHA256: [u8; 32] = [ 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; -/// T8122 C0, firmware-compat 26.6.2. Selection remains separate from runtime admission. +/// T8122 C0, firmware-compat 26.6.2 segment and loader-mutable bounds. pub(crate) static T8122_LAYOUT_26: Layout = Layout { text_size: 393216, data_size: 1228800, @@ -313,7 +313,10 @@ pub(crate) fn identify_loaded( prelude::*, }; - let layout = soc.firmware.ok_or(ENODEV)?; + let extended=crate::m3::t8122_start::is_t8122(soc) + && crate::m3::t8122_admission::loaded_profile(pdev)?==super::m3_profile::Profile::Extended; + if extended && experiment.is_some() {return Err(EINVAL);} + let layout = if extended {&T8122_LAYOUT_26} else {soc.firmware.ok_or(ENODEV)?}; if !Firmware::matching_layout(layout, &resources) { dev_err!(pdev.as_ref(), "M3 {}: unsupported firmware segment layout\n", soc.gpu_name); return Err(ENODEV); @@ -337,6 +340,7 @@ pub(crate) fn identify_loaded( // The T8122 start experiment identifies the same image, with its own InitData version. let images = match experiment { Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..], + None if extended => &T8122_IMAGES_26[..], None => soc.images, }; let image = crate::m3::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs b/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs index 44671ee3cd0ab..e0aaf2584284a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs +++ b/drivers/gpu/drm/asahi/m3/m3_fragment_command.rs @@ -3,6 +3,8 @@ use super::{agx_render::Geometry, agx_render_state::{Program,DepthStencil}, m3_compute_layout::{GpuVa,RegisterSet}, m3_init_layout::{Error,Region}, m3_queue_layout::FirmwareVa, m3_render_sequence::fragment}; pub(crate) const SIZE:usize=0xc73; +pub(crate) const EXTENDED_SIZE:usize=0xca0; +pub(crate) const EXTENDED_FLUSH:usize=0xbdc; pub(crate) const REGISTERS:usize=fragment::REGISTERS; /// The registers G15G does not take; the others keep the G15S order. pub(crate) const G15G_OMIT:[u32;4]=[0x160a8,0x160b8,0xa5a9,0xd429]; @@ -74,16 +76,30 @@ impl Command { /// Check references before touching output. The caller owns validation of /// client resource mappings; this layer serializes their semantic state. pub(crate) fn encode(&self,out:&mut[u8])->Result<(),Error> { - if out.len()Result<(),Error> { + self.encode_format(out,Some(dispatch)) + } + fn encode_format(&self,out:&mut[u8],dispatch:Option)->Result<(),Error> { + let extended=dispatch.is_some(); + let size=if extended {EXTENDED_SIZE} else {SIZE}; + if out.len()=1<<42 || self.register_set!=RegisterSet::G15G {return Err(Error::Address);} + } if self.context>=64 {return Err(Error::Address);} - let seq=self.sequence.at(0,fragment::LENGTH)?; + let sequence_size=if extended {0x370} else {fragment::LENGTH}; + let seq=self.sequence.at(0,sequence_size)?; let alias=self.gpu_alias.get(); - GpuVa::new(alias.checked_add((SIZE-3) as u64).ok_or(Error::Address)?).map_err(|_|Error::Address)?; + GpuVa::new(alias.checked_add(((size-1)&!3) as u64).ok_or(Error::Address)?).map_err(|_|Error::Address)?; let registers=alias+REGISTERS as u64; let s=&self.state;let g=&s.geometry; let layer_metadata=self.heap.get().checked_add(if g.layers>1 {0x200}else{0}).ok_or(Error::Address)?; GpuVa::new(layer_metadata).map_err(|_|Error::Address)?; - let user=super::agx_render_state::compact(self.scene_user.get().checked_add(0x40).ok_or(Error::Address)?) + let user=super::agx_render_state::compact(self.scene_user.get().checked_add(if extended {0} else {0x40}).ok_or(Error::Address)?) .map_err(|_|Error::Address)? & !15; let width=(g.pixels&0xffff)+1;let height=(g.pixels>>16)+1; let tx=width.div_ceil(32);let ty=height.div_ceil(32); @@ -116,43 +132,85 @@ impl Command { (0x101c1,0),(0xd469,0),(0x1a0f9,8),(0x1c838,0),(0x1ca28,user), (0xa5a9,0x6000400020),(0xd429,0x200000001), ]; - let bytes=&mut out[..SIZE];bytes.fill(0); + let bytes=&mut out[..size];bytes.fill(0); word(bytes,0,1); // fragment opcode; work counter remains zero word(bytes,0xc,self.context); for (offset,value) in [(0x14,seq),(0x20,self.notifier.get()),(0x28,self.manager.get()), (0x30,self.scene.get()),(0x38,self.empty.get()),(0x40,self.tilemap.get()),(0x780,registers)] {pointer(bytes,offset,value);} - word(bytes,SEQUENCE_LENGTH,fragment::LENGTH as u32); - // The qualified patch path updates register MSAA but retains these - // producer header values. Representation cleanup must not change that. - pointer(bytes,0x48,0x88);word(bytes,0x50,1); - bytes[0x54..0x56].copy_from_slice(&((g.y_blocks>>18) as u16).to_le_bytes()); - bytes[0x56..0x58].copy_from_slice(&((g.x_blocks>>18) as u16).to_le_bytes()); - word(bytes,0x68,s.merge_upper[0]);word(bytes,0x6c,s.merge_upper[1]); - pointer(bytes,0x78,u64::from(g.tiles)); + word(bytes,SEQUENCE_LENGTH,sequence_size as u32); + if extended { + pointer(bytes,4,dispatch.as_ref().map_or(0,|native|native.serial)); + word(bytes,0x50,1); + word(bytes,0x54,g.isp_blocks); + word(bytes,0x60,g.x_blocks); + word(bytes,0x64,g.y_blocks); + word(bytes,0x68,g.blocks); + word(bytes,0x78,g.tiles); + } else { + pointer(bytes,0x48,0x88);word(bytes,0x50,1); + bytes[0x54..0x56].copy_from_slice(&((g.y_blocks>>18) as u16).to_le_bytes()); + bytes[0x56..0x58].copy_from_slice(&((g.x_blocks>>18) as u16).to_le_bytes()); + word(bytes,0x68,s.merge_upper[0]);word(bytes,0x6c,s.merge_upper[1]); + pointer(bytes,0x78,u64::from(g.tiles)); + } let g15g=self.register_set==RegisterSet::G15G; let mut count=0; - for (register,value) in values.into_iter().filter(|(r,_)| !g15g || !G15G_OMIT.contains(r)) { - let n=count;count+=1; - word(bytes,REGISTERS+n*REGISTER_STRIDE,register);pointer(bytes,REGISTERS+n*REGISTER_STRIDE+4,value); + let mut emit=|register,value| { + word(bytes,REGISTERS+count*REGISTER_STRIDE,register); + pointer(bytes,REGISTERS+count*REGISTER_STRIDE+4,value); + count+=1; + }; + for (register,mut value) in values { + if extended { + if matches!(register,0x160b8|0x120a1|0x1c838|0x1ca28) {continue;} + if register==0x10791 {emit(0xa301,0);} + if register==0x16068 {value=(value&!0x3ffff)|0x3717f;} + if matches!(register,0x15109|0x15101) {value&=!3;} + } else if g15g && G15G_OMIT.contains(®ister) {continue;} + emit(register,value); + } + if let Some(native)=&dispatch { + for register in [0x160e0,0x1499,0xa341] {emit(register,fragment::UUID.into());} + emit(0x1c838,0); + emit(0x1ca28,user); + emit(0x10069,native.usc_window); } bytes[0x788..0x78a].copy_from_slice(&(count as u16).to_le_bytes()); bytes[0x78a..0x78c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes()); // Start3DJobParameters3: mirror records use their actual packed widths. for (offset,value) in [(0,s.depth_bias),(0x10,s.scissor),(0x20,s.query), - (0x168,s.partial_background.resources),(0x170,s.partial_background.address), + (0x168,if extended {s.background.resources}else{s.partial_background.resources}), + (0x170,if extended {s.background.address}else{s.partial_background.address}), (0x198,s.partial_background.resources),(0x1a0,s.partial_background.address), (0x1a8,s.zls_control),(0x1b8,s.depth.base),(0x1c0,s.depth.stride), (0x1c8,s.depth_compression.stride),(0x1e0,s.depth_compression.base), (0x1d0,s.depth.base),(0x1d8,s.depth.base), (0x1e8,s.stencil.base),(0x1f0,s.stencil.stride),(0x200,s.stencil.base),(0x208,s.stencil.base), (0x1f8,s.stencil_compression.stride),(0x210,s.stencil_compression.base), - (0x26c,s.partial_eot.address),(0x28c,s.partial_eot.address),(0x2c0,u64::from(s.depth_dimensions))] { - pointer(bytes,PARAMETERS+offset,value); + (0x26c,if extended {s.eot.address}else{s.partial_eot.address}),(0x28c,s.partial_eot.address),(0x2c0,u64::from(s.depth_dimensions))] { + pointer(bytes,PARAMETERS+offset+if extended && offset<=0x20 {0x10}else{0},value); } for (offset,value) in [(0x1b0,0x04040404),(0x228,s.mirror_tilebuffer_control), (0x230,s.isp_control),(0x238,width),(0x23c,height),(0x240,0x100000), - (0x248,s.mirror_tile_mode),(0x264,s.partial_eot.resources as u32),(0x284,s.partial_eot.resources as u32), + (0x248,s.mirror_tile_mode),(0x264,if extended {s.eot.resources as u32}else{s.partial_eot.resources as u32}),(0x284,s.partial_eot.resources as u32), (0x298,s.depth_clear),(0x29c,s.stencil_clear),(0x2a0,s.sample_size)] {word(bytes,PARAMETERS+offset,value);} + if let Some(native)=dispatch { + pointer(bytes,0xbc0,self.stamp.get()); + pointer(bytes,0xbc8,self.fw_stamp.get()); + word(bytes,0xbd0,self.stamp_value); + word(bytes,0xbd4,1); + word(bytes,EXTENDED_FLUSH,1); + word(bytes,0xbe0,fragment::UUID); + pointer(bytes,0xc10,self.timestamps[0].get()); + pointer(bytes,0xc18,self.timestamps[1].get()); + pointer(bytes,0xc20,self.user_timestamps[0].get()); + pointer(bytes,0xc28,self.user_timestamps[1].get()); + pointer(bytes,0xc46,self.pool.get()); + bytes[0xc4e]=1; + pointer(bytes,0xc57,native.firmware_aux.get()); + bytes[0xc5f]=native.incarnation; + bytes[0xc84]=1; + } else { word(bytes,0xb88,0x67); // EncoderParams.encoder_id word(bytes,0xb90,u32::MAX); // EncoderParams.unk_mask word(bytes,0xb94,u32::from(s.process_empty_tiles)); @@ -167,6 +225,46 @@ impl Command { pointer(bytes,fragment::COMMAND_TAIL+offset,value); } bytes[fragment::COMMAND_TAIL+0x36]=1;bytes[fragment::COMMAND_TAIL+0x4f]=1; + } Ok(()) } } + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] + fn native_fragment_uses_distinct_program_mirrors_and_completion_tail() { + let fw=|n:u64|FirmwareVa::new(0xffff_fc25_3000_0000+n*0x4000).unwrap(); + let gpu=|n:u64|GpuVa::new(0x10_0300_0000+n*0x4000).unwrap(); + let program=|n:u64|Program{address:0x1_0000_0000+n*0x1000,resources:n}; + let surface=DepthStencil{base:0,stride:0}; + let command=Command {context:3,register_set:RegisterSet::G15G, + sequence:Region::new(fw(0).get(),0x1000).unwrap(),notifier:fw(1),manager:fw(2),scene:fw(3),empty:fw(4), + gpu_alias:GpuVa::new(0x72_1000_4000).unwrap(),tilemap:gpu(0),heap:gpu(1),auxiliary:gpu(2),scene_user:gpu(3), + scratch:gpu(4),pool:fw(5),stamp:fw(6),fw_stamp:fw(7),stamp_value:0x3d000100, + timestamps:[fw(8),fw(9)],user_timestamps:[fw(10),fw(11)], + state:State {geometry:Geometry::new(64,64,super::super::agx_render::Utile::Pixels32, + super::super::agx_render::Utile::Pixels32,1).unwrap(),multisample:0,merge_upper:[0;2], + eot:program(1),background:program(2),partial_eot:program(3),partial_background:program(4), + scissor:0,depth_bias:0,query:0,isp_control:0xc000,tilebuffer_control:1,mirror_tilebuffer_control:1, + tile_mode:0x280,mirror_tile_mode:0x280,depth:surface,stencil:surface,depth_compression:surface, + stencil_compression:surface,zls_control:0,depth_dimensions:0,depth_clear:0,stencil_clear:0, + sample_size:4,process_empty_tiles:false}}; + let dispatch=||super::super::m3_tiler_command::Dispatch26 {incarnation:9,serial:1,peer_stamp:0, + usc_window:1<<32,firmware_aux:fw(12),scene_scratch:gpu(4)}; + let mut bytes=[0xa5;EXTENDED_SIZE+8]; + command.encode_extended(dispatch(),&mut bytes).unwrap(); + let word=|at|u32::from_le_bytes(bytes[at..at+4].try_into().unwrap()); + let pointer=|at|u64::from_le_bytes(bytes[at..at+8].try_into().unwrap()); + assert_eq!(word(0xbd0),command.stamp_value); + assert_eq!(word(EXTENDED_FLUSH),1); + assert_eq!(pointer(0x900),command.state.background.address); + assert_eq!(pointer(0x930),command.state.partial_background.address); + assert_eq!(bytes[0xc5f],9); + assert_eq!(&bytes[EXTENDED_SIZE..],&[0xa5;8]); + let mut short=[0xa5;EXTENDED_SIZE-1]; + assert!(command.encode_extended(dispatch(),&mut short).is_err()); + assert!(short.iter().all(|b|*b==0xa5)); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs b/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs index 684505615adf3..97c1367b29455 100644 --- a/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs @@ -61,3 +61,77 @@ impl Manager { Ok(()) } } + +impl Manager { + pub(crate) const EXTENDED_SIZE: usize = 0xc0; + + /// J613's extended manager has both firmware and GPU page-list aliases. + /// The block ring is read through its canonical firmware mapping. + pub(crate) fn encode_extended(&self, pages_fw: FirmwareVa, blocks_fw: FirmwareVa, + hardware_id: u32, output: &mut [u8]) -> Result<(), Error> { + let capacity = self.block_count.checked_add(1).and_then(u32::checked_next_power_of_two) + .ok_or(Error::Size)?; + if hardware_id >= 32 || capacity as usize > self.blocks.size() / 8 { + return Err(Error::Size); + } + let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + let word = |bytes: &mut [u8], offset: usize, value: u32| { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + }; + let address = |bytes: &mut [u8], offset: usize, value: u64| { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + }; + word(bytes, 8, hardware_id); + word(bytes, 12, hardware_id); + address(bytes, 0x1c, pages_fw.get()); + address(bytes, 0x24, self.pages.base()); + word(bytes, 0x2c, self.pages.size() as u32); + word(bytes, 0x30, self.page_count); + word(bytes, 0x34, capacity); + word(bytes, 0x38, self.block_count); + address(bytes, 0x40, blocks_fw.get()); + address(bytes, 0x48, self.control.get()); + word(bytes, 0x50, self.page_count - 1); + word(bytes, 0x54, BLOCK_SIZE as u32); + address(bytes, 0x60, self.counter.get()); + word(bytes, 0x78, self.page_count); + word(bytes, 0x7c, self.page_count); + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + fn fixture() -> Manager { + Manager::new(GpuRegion::new(0x5000_0000, 0x4000).unwrap(), + GpuRegion::new(0x6000_0000, 0x4000).unwrap(), + FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), + FirmwareVa::new(0xffff_fc24_0020_0000).unwrap(), 32).unwrap() + } + #[test] fn native_manager_keeps_distinct_aliases_and_hardware_identity() { + let mut bytes = [0xa5; Manager::EXTENDED_SIZE + 8]; + let pages = FirmwareVa::new(0xffff_fc24_0030_0000).unwrap(); + let blocks = FirmwareVa::new(0xffff_fc24_0040_0000).unwrap(); + fixture().encode_extended(pages, blocks, 7, &mut bytes).unwrap(); + let word = |at| u32::from_le_bytes(bytes[at..at + 4].try_into().unwrap()); + let pointer = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()); + assert_eq!(word(8), 7); assert_eq!(word(12), 7); + assert_eq!(pointer(0x1c), pages.get()); + assert_eq!(pointer(0x24), 0x5000_0000); + assert_eq!(pointer(0x40), blocks.get()); + assert_eq!(word(0x34), 64); assert_eq!(word(0x38), 32); + assert_eq!(word(0x50), 127); + assert_eq!(pointer(0x80), 0); + assert_eq!(&bytes[Manager::EXTENDED_SIZE..], &[0xa5; 8]); + } + #[test] fn native_manager_rejects_short_storage_and_invalid_identity_before_mutation() { + let fw = FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(); + let mut bytes = [0xa5; Manager::EXTENDED_SIZE]; + assert!(fixture().encode_extended(fw, fw, 32, &mut bytes).is_err()); + assert_eq!(bytes, [0xa5; Manager::EXTENDED_SIZE]); + assert!(fixture().encode_extended(fw, fw, 0, &mut bytes[..MANAGER_SIZE]).is_err()); + assert_eq!(bytes, [0xa5; Manager::EXTENDED_SIZE]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs index 94c2172e33279..9c0eb070039cd 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs @@ -8,10 +8,10 @@ use layout::{Field,Space}; pub(crate) struct Pass {buffers:KVec} impl Pass { pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm, - slot:usize,clusters:u32,aliases:&mut KVec)->Result { + slot:usize,clusters:u32,profile:super::m3_profile::Profile,aliases:&mut KVec)->Result { let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?; for field in layout::FIELDS { - let a=layout::board_allocation(slot,field,clusters).map_err(|_|EINVAL)?; + let a=layout::allocation_for(profile,slot,field,clusters).map_err(|_|EINVAL)?; let mut b=match a.space { Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, a.size,prot::PROT_GPU_FW_SHARED_RW,prot::Prot::from_pte(a.access.pte()))?, diff --git a/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs b/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs index 5ace477a44fff..f88a2a2c9b4a5 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass_layout.rs @@ -219,3 +219,39 @@ mod tests { } } } + +pub(crate) fn allocation_for(profile: super::m3_profile::Profile, slot:usize, field:Field, clusters:u32) + -> Result { + let mut owner = board_allocation(slot,field,clusters)?; + if profile == super::m3_profile::Profile::Legacy { return Ok(owner); } + if owner.space == Space::Firmware { + owner.address = 0xffff_fc25_0000_0000 + (slot * COUNT + field as usize) as u64 * 0x10_0000; + } + match field { + Field::Scene => owner.size = 0x400, + Field::TilerCommand | Field::FragmentCommand => { + let fragment = field == Field::FragmentCommand; + owner.size = if fragment {0xca0} else {0x940}; + owner.gpu_alias = Some(0x72_1000_0000 + slot as u64 * 0x4_0000 + + u64::from(fragment) * 0x4000); + } + Field::TilerToFragment | Field::TilerDependency | Field::FragmentDependency => owner.size = 0x40, + _ => {} + } + Ok(owner) +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn extended_command_aliases_keep_whole_disjoint_pages() { + for slot in 0..SLOTS { + for field in [Field::TilerCommand,Field::FragmentCommand] { + let owner = allocation_for(super::super::m3_profile::Profile::Extended,slot,field,1).unwrap(); + assert_eq!(owner.address & 0x3fff,0); + assert_eq!(owner.gpu_alias.unwrap() & 0x3fff,0); + assert!(owner.size <= 0x4000); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_profile.rs b/drivers/gpu/drm/asahi/m3/m3_profile.rs index 004ac2c7d1eef..04b872ed9ea80 100644 --- a/drivers/gpu/drm/asahi/m3/m3_profile.rs +++ b/drivers/gpu/drm/asahi/m3/m3_profile.rs @@ -32,6 +32,15 @@ impl Profile { } } + /// Keep the diagnostic AP/GPU timeline in its established 24 MHz units. + /// User timestamps and duration calculations retain native firmware units. + pub(crate) const fn profiler_ticks(self, timestamp: u64) -> u64 { + match self { + Self::Legacy => timestamp, + Self::Extended => (timestamp / 1000) * 24 + (timestamp % 1000) * 24 / 1000, + } + } + pub(crate) fn gpu_duration_ns(self, elapsed: u64) -> Option { match self { Self::Legacy => elapsed.checked_mul(1000).map(|value| value / 24), @@ -49,6 +58,18 @@ impl Profile { mod tests { use super::*; + #[test] + fn profiling_uses_one_clock_domain_without_changing_native_durations() { + for value in [0, 1, 999, 1000, 24_000_000, u64::MAX] { + assert_eq!(Profile::Legacy.profiler_ticks(value), value); + assert_eq!( + Profile::Extended.profiler_ticks(value), + ((value as u128) * 24 / 1000) as u64 + ); + } + assert_eq!(Profile::Extended.profiler_ticks(1_000_000_000), 24_000_000); + } + #[test] fn profile_acknowledgement_is_exact_and_file_query_covers_hal() { for profile in [Profile::Legacy, Profile::Extended] { diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs index e7a02982e96f1..9a9f63d15b784 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -26,7 +26,7 @@ pub(crate) struct Render { passes:KVec, init_bm:Buffer, // All active mappings above must drop before their ASID binding. _binding:mmu::VmBind, - stats: Region, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, + stats: [Region;2], profile:super::m3_profile::Profile, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, last_progress_ns:i64, draw:u64, heads:[u16;2], cached_views:KVec, slot:usize, batch_count:usize, early_count:u32, @@ -67,37 +67,45 @@ const RENDER_QUEUES: [QueueObjects; 2] = [ scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 }, ]; impl Render { - pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, - registers:crate::m3::m3_compute_layout::RegisterSet)->Result { + pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:[Region;2],clusters:u32, + registers:crate::m3::m3_compute_layout::RegisterSet,profile:super::m3_profile::Profile)->Result { let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new();let mut aliases=KVec::new(); for index in 0..s::COUNT { - let a=s::allocation(index).map_err(|_|EINVAL)?; + let a=s::allocation_for(profile,index).map_err(|_|EINVAL)?; let protection=prot::Prot::from_pte(a.access.pte()); let b=match a.space { Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, a.size,prot::PROT_GPU_FW_SHARED_RW,protection)?, - Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address), + Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(), + a.gpu_alias.map(|address|(uat.kernel_lower_vm(),address)),Some(a.address), a.size,protection,prot::PROT_GPU_SHARED_RW)?, }; + let mut b=b; + if a.gpu_alias.is_some() { aliases.push(b.map_gpu_view(vm)?,GFP_KERNEL)?; } objects.push(b,GFP_KERNEL)?; } // InitBM is shared, not part of a pass. Preserve its qualified mapping. - let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(0xfffffc200062bfe0), + let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(if profile==super::m3_profile::Profile::Extended {0xffff_fc24_4000_0000} + else {0xfffffc200062bfe0}), sync::INIT_BM_SIZE,prot::PROT_FW_SHARED_RW,prot::PROT_GPU_SHARED_RW)?; let mut passes=KVec::with_capacity(pass_layout::SLOTS,GFP_KERNEL)?; - for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,&mut aliases)?,GFP_KERNEL)?;} + for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,profile,&mut aliases)?,GFP_KERNEL)?;} let mut sequence_bytes=KVec::new(); sequence_bytes.resize(sequence::STORAGE,0u8,GFP_KERNEL)?; let mut tiler_bytes=KVec::new(); - tiler_bytes.resize(tiler::SIZE,0u8,GFP_KERNEL)?; + tiler_bytes.resize(if profile==super::m3_profile::Profile::Extended {tiler::EXTENDED_SIZE} else {tiler::SIZE},0u8,GFP_KERNEL)?; let mut fragment_bytes=KVec::new(); - fragment_bytes.resize(fragment::SIZE,0u8,GFP_KERNEL)?; - let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; + fragment_bytes.resize(if profile==super::m3_profile::Profile::Extended {fragment::EXTENDED_SIZE} else {fragment::SIZE},0u8,GFP_KERNEL)?; + let mut job=Self{passes,init_bm,stats,profile,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; // Lists and the matching manager are built before InitBM publication. job.initialize_parameter_buffer()?; - for pool in &UMA_POOLS { pool.initialize(&mut job.objects)?; } + for (index,pool) in UMA_POOLS.iter().enumerate() { + if profile==super::m3_profile::Profile::Extended { + pool.initialize_extended(&mut job.objects,33+index as u8,false)?; + } else {pool.initialize(&mut job.objects)?;} + } job.initialize_state()?; job.initialize_queues()?; dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024)); @@ -113,8 +121,14 @@ impl Render { let control = queue::FirmwareVa::new(self.objects[BLOCK_CONTROL].va()).map_err(|_| EINVAL)?; let counter = queue::FirmwareVa::new(self.objects[s::BM_COUNTER].va()).map_err(|_| EINVAL)?; let manager = parameter::Manager::new(pages, blocks, control, counter, PB_BLOCKS).map_err(|_| EINVAL)?; - let mut bytes = [0; parameter::MANAGER_SIZE]; - manager.encode(&mut bytes).map_err(|_| EINVAL)?; + let extended=self.profile==super::m3_profile::Profile::Extended; + let mut bytes = [0; parameter::Manager::EXTENDED_SIZE]; + let length=if extended { + manager.encode_extended(queue::FirmwareVa::new(self.objects[PARAMETER_PAGE_LIST].va()).map_err(|_|EINVAL)?, + queue::FirmwareVa::new(self.objects[PARAMETER_BLOCK_LIST].va()).map_err(|_|EINVAL)?,0,&mut bytes) + .map_err(|_|EINVAL)?; + parameter::Manager::EXTENDED_SIZE + } else {manager.encode(&mut bytes).map_err(|_|EINVAL)?;parameter::MANAGER_SIZE}; for group in 0..PB_GROUPS { let buffer = &self.objects[PB_FIRST + group]; let backing = parameter::Backing::new(GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?) @@ -128,18 +142,30 @@ impl Render { backing.page(page).map_err(|_| EINVAL)?)?; } } - self.objects[BUFFER_MANAGER].write(0, &bytes) + self.objects[BUFFER_MANAGER].write(0, &bytes[..length]) } fn initialize_state(&mut self) -> Result { let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL); let notification = state::Notifier { threshold: address(s::EVENT_COUNT)?, context: self._binding.slot() }; let list_owner = address(JOB_LIST)?; let mut context = [0; state::CONTEXT_SIZE]; - state::context(&mut context).map_err(|_| EINVAL)?; - self.objects[GPU_CONTEXT].write(0, &context)?; - let mut notifier = [0; state::NOTIFIER_SIZE]; - notification.encode(&mut notifier).map_err(|_| EINVAL)?; - self.objects[NOTIFIER].write(0, ¬ifier)?; + if self.profile==super::m3_profile::Profile::Extended { + state::context_extended(&mut context).map_err(|_|EINVAL)?; + self.objects[GPU_CONTEXT].write(0,&context[..state::EXTENDED_CONTEXT_SIZE])?; + for engine in 0..2 { + let value=state::Notifier {threshold:queue::FirmwareVa::new(self.objects[s::EVENT_COUNT].va()+engine as u64*8).map_err(|_|EINVAL)?, + context:self._binding.slot()}; + let mut notifier=[0;state::EXTENDED_NOTIFIER_SIZE]; + value.encode_extended(&mut notifier).map_err(|_|EINVAL)?; + self.objects[NOTIFIER].write(engine*state::EXTENDED_NOTIFIER_SIZE,¬ifier)?; + } + } else { + state::context(&mut context).map_err(|_| EINVAL)?; + self.objects[GPU_CONTEXT].write(0, &context)?; + let mut notifier = [0; state::NOTIFIER_SIZE]; + notification.encode(&mut notifier).map_err(|_| EINVAL)?; + self.objects[NOTIFIER].write(0, ¬ifier)?; + } let mut list = [0; state::JOB_LIST_SIZE]; state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; self.objects[JOB_LIST].write(0, &list)?; @@ -157,6 +183,10 @@ impl Render { state::counter(bytes, value).map_err(|_| EINVAL)?; self.objects[owner].write(0, bytes)?; } + if self.profile==super::m3_profile::Profile::Extended { + self.objects[s::EVENT_COUNT].fill(0)?; + self.objects[s::BM_COUNTER].fill(0)?; + } Ok(()) } fn initialize_queues(&mut self) -> Result { @@ -172,8 +202,13 @@ impl Render { }else{[self.passes[0].get(P::TilerToFragment).va(),self.passes[0].get(P::FragmentCommand).va()]}; let seeds=[queue::FirmwareVa::new(seed_addresses[0]).map_err(|_|EINVAL)?, queue::FirmwareVa::new(seed_addresses[1]).map_err(|_|EINVAL)?]; - let mut bytes = [0; queue::INFO_SIZE]; - info.encode(&mut bytes).map_err(|_| EINVAL)?; + let mut bytes=KVec::new(); + bytes.resize(if self.profile==super::m3_profile::Profile::Extended { + queue::QueueInfo::EXTENDED_SIZE + } else {queue::INFO_SIZE},0u8,GFP_KERNEL)?; + if self.profile==super::m3_profile::Profile::Extended { + info.encode_extended(address(owner.info)?,&mut bytes).map_err(|_|EINVAL)?; + } else {info.encode(&mut bytes).map_err(|_|EINVAL)?;} self.objects[owner.info].write(0, &bytes)?; let mut state = [0; queue::STATE_SIZE]; queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 0 } @@ -187,9 +222,12 @@ impl Render { Ok(()) } fn write_barrier(&mut self, owner: P, command: sync::Barrier) -> Result { - let mut bytes = [0; sync::BARRIER_SIZE]; - command.encode(&mut bytes).map_err(|_| EINVAL)?; - self.passes[self.slot].get_mut(owner).write(0, &bytes) + let mut bytes=[0;sync::Barrier::EXTENDED_SIZE]; + let length=if self.profile==super::m3_profile::Profile::Extended { + command.encode_extended(&mut bytes).map_err(|_|EINVAL)?; + sync::Barrier::EXTENDED_SIZE + } else {command.encode(&mut bytes).map_err(|_|EINVAL)?;sync::BARRIER_SIZE}; + self.passes[self.slot].get_mut(owner).write(0,&bytes[..length]) } // Construct synchronization from the actual owners after resetting a pass, // before publishing its ring entries. InitBM is shared and dispatched only @@ -201,9 +239,11 @@ impl Render { let ta_stamp = address(s::TA_FW_STAMP)?; let fs_stamp = address(s::FRAGMENT_FW_STAMP)?; let mut init = [0; sync::INIT_BM_SIZE]; - sync::InitBufferManager { context: self._binding.slot(), slot: 0, - block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) } - .encode(&mut init).map_err(|_| EINVAL)?; + let value=sync::InitBufferManager { context: self._binding.slot(), slot: 0, + block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) }; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(&mut init).map_err(|_|EINVAL)?; + } else {value.encode(&mut init).map_err(|_|EINVAL)?;} self.init_bm.write(0, &init)?; let ta_value = timeline::stamp(s::STAMP_TA - 256, draw); let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw); @@ -225,7 +265,7 @@ impl Render { } // Prepare each private command after its retired slot is reset. The // reusable host staging allocation is copied before the next slot is built. - fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,draw:u64)->Result { + fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result { use crate::m3::m3_compute_layout::GpuVa; let pass=&self.passes[self.slot]; let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); @@ -243,7 +283,7 @@ impl Render { let value=tiler::Command { context:self._binding.slot(),register_set:self.registers, notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?, - scene:fw(P::Scene)?,empty:shared(547)?,gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, + scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?},gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?, scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?, stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], @@ -253,7 +293,12 @@ impl Render { vdm:crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0, }; - value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).ok_or(EOVERFLOW)?, + peer_stamp:timeline::stamp(s::STAMP_FRAGMENT-256,draw),usc_window:usc, + firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[0].manager].va()+0x100).map_err(|_|EINVAL)?, + scene_scratch:gpu(P::FragmentScratch)?},&mut self.tiler_bytes).map_err(|_|EINVAL)?; + } else {value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?;} self.passes[self.slot].get_mut(P::TilerCommand).write(0,&self.tiler_bytes) } fn prepare_sequences(&mut self,draw:u64)->Result { @@ -266,14 +311,21 @@ impl Render { let notifier=&self.objects[NOTIFIER]; let owners=sequence::Owners { command:region(if fs{P::FragmentCommand}else{P::TilerCommand})?,sequence:region(target)?, - stats:self.stats,notifier:Region::new(notifier.va(),notifier.size()).map_err(|_|EINVAL)?, + stats:self.stats[usize::from(fs)],notifier:Region::new(notifier.va()+ + if fs && self.profile==super::m3_profile::Profile::Extended {0x200} else {0}, + if self.profile==super::m3_profile::Profile::Extended {0x200} else {notifier.size()}).map_err(|_|EINVAL)?, queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?, manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?, fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?, context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw), }; let timed=self.stage_timestamps[usize::from(fs)]; - let length=if fs { + let length=if self.profile==super::m3_profile::Profile::Extended { + let pool=&self.objects[UMA_POOLS[usize::from(fs)].manager]; + let auxiliary=super::m3_compute_layout::GpuVa::new(pool.gpu_va()?+if fs{0x200}else{0x100}).map_err(|_|EINVAL)?; + let serial=draw.checked_mul(2).and_then(|v|v.checked_sub(u64::from(fs))).ok_or(EOVERFLOW)?; + owners.encode_extended(fs,auxiliary,serial,&mut self.sequence_bytes).map_err(|_|EINVAL)? + } else if fs { let b=pass.get(P::FragmentScratch); let scratch=GpuRegion::new(b.gpu_va()?,b.size()).map_err(|_|EINVAL)?; sequence::Fragment{owners,scratch,queue_command_count:timeline::previous(draw)}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)? @@ -290,6 +342,9 @@ impl Render { fn patch_context(&mut self)->Result { let context=self._binding.slot(); self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT, context)?; + if self.profile==super::m3_profile::Profile::Extended { + self.objects[NOTIFIER].u32(0x200+state::NOTIFIER_CONTEXT,context)?; + } Ok(()) } /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster. @@ -320,7 +375,8 @@ impl Render { let value=fragment::Command { context:self._binding.slot(),register_set:self.registers, sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?, - notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:shared(547)?, + notifier:queue::FirmwareVa::new(self.objects[NOTIFIER].va()+ + if self.profile==super::m3_profile::Profile::Extended {0x200} else {0}).map_err(|_|EINVAL)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:if self.profile==super::m3_profile::Profile::Extended {queue::FirmwareVa::new(pass.get(P::Scene).va()+0x100).map_err(|_|EINVAL)?} else {shared(547)?}, gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?, heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?, scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?, @@ -341,15 +397,34 @@ impl Render { stencil_clear:r.stencil_clear,sample_size:u32::from(r.sample_size),process_empty_tiles:r.flags&2!=0, }, }; - value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?; + if self.profile==super::m3_profile::Profile::Extended { + value.encode_extended(tiler::Dispatch26 {incarnation:self._binding.generation(),serial:draw.checked_mul(2).and_then(|v|v.checked_sub(1)).ok_or(EOVERFLOW)?, + peer_stamp:0,usc_window:usc, + firmware_aux:queue::FirmwareVa::new(self.objects[UMA_POOLS[1].manager].va()+0x200).map_err(|_|EINVAL)?, + scene_scratch:gpu(P::FragmentScratch)?},&mut self.fragment_bytes).map_err(|_|EINVAL)?; + } else {value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?;} self.passes[self.slot].get_mut(P::FragmentCommand).write(0,&self.fragment_bytes) } + pub(crate) fn set_attachments(&mut self,attachments:&[crate::m3::agx_attachments::Attachments;2])->Result { + // Preserve the legacy renderer's existing packet format. The native + // interface consumes separate attachment apertures for both stages. + if self.profile!=super::m3_profile::Profile::Extended {return Ok(());} + for (fragment,list) in attachments.iter().enumerate() { + let field=if fragment==0 {P::TilerSequence}else{P::FragmentSequence}; + let buffer=self.passes[self.slot].get_mut(field); + sequence::patch_extended_attachments(fragment!=0,list,|offset,data|buffer.write(offset,data))?; + } + crate::m3::agx_memory::publish();Ok(()) + } pub(crate) fn set_user_timestamps(&mut self,addresses:[[u64;2];2])->Result { if addresses.iter().enumerate().any(|(stage,pair)|*pair!=[0;2] && !self.stage_timestamps[stage]) {return Err(EINVAL);} for (stage,(command,sequence,offset,start,end)) in [ (P::TilerCommand,P::TilerSequence,sequence::ta::USER_PAIR,sequence::ta::TIMESTAMP_START,sequence::ta::TIMESTAMP_END), (P::FragmentCommand,P::FragmentSequence,sequence::fragment::USER_PAIR,sequence::fragment::TIMESTAMP_START,sequence::fragment::TIMESTAMP_END) ].into_iter().enumerate() { + let (offset,start,end)=if self.profile==super::m3_profile::Profile::Extended { + if stage==0 {(0x8d8,0x1cc,0x21c)} else {(0xc20,0x1e4,0x234)} + } else {(offset,start,end)}; let pair=addresses[stage]; if !self.stage_timestamps[stage] {continue;} self.passes[self.slot].get_mut(command).u64(offset,pair[0])?; @@ -365,19 +440,19 @@ impl Render { pub(crate) fn gpu_ns(&mut self,base:usize,count:usize)->Result { let start=self.passes[base].get_mut(P::TilerStart).read_u64(0)?; let end=self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?; - Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + self.profile.gpu_duration_ns(if start!=0 && end>=start {end-start} else {0}).ok_or(EOVERFLOW) } /// The batch's first tiler start and last fragment end, in 24 MHz GPU timestamp ticks. pub(crate) fn gpu_ticks(&mut self,base:usize,count:usize)->Result<(u64,u64)> { - Ok((self.passes[base].get_mut(P::TilerStart).read_u64(0)?, - self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?)) + Ok((self.profile.profiler_ticks(self.passes[base].get_mut(P::TilerStart).read_u64(0)?), + self.profile.profiler_ticks(self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?))) } /// Read retained timestamps only after exact batch retirement. The first /// TA and final fragment delimit the batch in the shared 24 MHz counter. pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { if self.batch_count == 0 {return Err(EINVAL);} - Ok([self.passes[self.base].get_mut(P::TilerStart).read_u64(0)?, - self.passes[Self::at(self.base,self.batch_count-1)].get_mut(P::FragmentEnd).read_u64(0)?]) + let (start,end)=self.gpu_ticks(self.base,self.batch_count)?; + Ok([start,end]) } pub(crate) fn stage_ns(&mut self,base:usize,count:usize)->Result<[u64;3]> { let mut total=[0;3]; @@ -390,7 +465,7 @@ impl Render { pub(crate) fn slot_stage_ns(&mut self,base:usize,count:usize,index:usize)->Result<[u64;3]> { if index>=count {return Err(EINVAL);} let slot=Self::at(base,index); - let duration=|a:u64,b:u64| if a!=0 && b>=a {(b-a)*1000/24} else {0}; + let duration=|a:u64,b:u64| self.profile.gpu_duration_ns(if a!=0 && b>=a {b-a} else {0}).unwrap_or(0); let fs_start=self.passes[slot].get_mut(P::FragmentStart).read_u64(0)?; let fs_end=self.passes[slot].get_mut(P::FragmentEnd).read_u64(0)?; let ta_start=self.passes[slot].get_mut(P::TilerStart).read_u64(0)?; @@ -415,7 +490,9 @@ impl Render { } /// Whether any draw has been published (the first one initializes the buffer manager). pub(crate) fn started(&self)->bool {self.draw>0} - pub(crate) fn first(&self)->bool {self.draw==self.batch_count as u64} + pub(crate) fn first(&self)->bool { + self.profile==super::m3_profile::Profile::Legacy && self.draw==self.batch_count as u64 + } /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then /// the VM must already be bound, no storage may grow and the slots must not be in flight /// (the caller keeps the in-flight passes plus this batch within SLOTS). @@ -447,13 +524,15 @@ impl Render { let binding=uat.bind(vm)?; let mut views=KVec::new();let mut aliases=KVec::new(); for i in 0..s::COUNT { - if s::allocation(i).map_err(|_|EINVAL)?.space==Space::ClientGpu { + let a=s::allocation_for(self.profile,i).map_err(|_|EINVAL)?; + if a.gpu_alias.is_some() {aliases.push(self.objects[i].map_gpu_view(vm)?,GFP_KERNEL)?;} + if a.space==Space::ClientGpu { views.push((ViewOwner::Shared(i),self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; } } for (slot,pass) in self.passes.iter_mut().enumerate() { for field in pass_layout::FIELDS { - let a=pass_layout::board_allocation(slot,field,self.clusters).map_err(|_|EINVAL)?; + let a=pass_layout::allocation_for(self.profile,slot,field,self.clusters).map_err(|_|EINVAL)?; if a.space==Space::ClientGpu {views.push((ViewOwner::Pass{slot,field},pass.get_mut(field).map_gpu_view(vm)?),GFP_KERNEL)?;} if a.gpu_alias.is_some() {aliases.push(pass.get_mut(field).map_gpu_view(vm)?,GFP_KERNEL)?;} } @@ -498,12 +577,22 @@ impl Render { self.base=base; Ok(()) } + fn publish_counts(&mut self, draw:u64) -> Result { + if self.profile==super::m3_profile::Profile::Extended { + for offset in [0,8] {self.objects[s::EVENT_COUNT].u64(offset,draw)?;} + for counter in [TA_POOL_COUNTER,FRAGMENT_POOL_COUNTER] {self.objects[counter].u64(0,draw)?;} + } else { + self.objects[s::BM_COUNTER].u32(0,draw as u32)?; + self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + } + Ok(()) + } + /// Restore producer state only when no batch is in flight. Running firmware can /// consume append's WRITE updates without a new doorbell, so overlap cannot abort. pub(crate) fn abort_batch(&mut self)->Result { (self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base)=self.checkpoint; - self.objects[s::BM_COUNTER].u32(0,self.draw as u32)?; - self.objects[s::EVENT_COUNT].u32(0,timeline::events(self.draw,2))?; + self.publish_counts(self.draw)?; for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() { self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; } @@ -547,8 +636,20 @@ impl Render { self.draw=self.draw.checked_add(1).ok_or(EOVERFLOW)?; self.batch_count+=1; let draw=self.draw; - self.objects[s::BM_COUNTER].u32(0,draw as u32)?; - self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + self.publish_counts(draw)?; + if self.profile==super::m3_profile::Profile::Extended { + let pass=&self.passes[self.slot]; + let scratch=pass.get(P::FragmentScratch); + let scratch_gpu=GpuRegion::new(scratch.gpu_va()?,scratch.size()).map_err(|_|EINVAL)?; + let scratch_fw=queue::FirmwareVa::new(scratch.va()).map_err(|_|EINVAL)?; + let list=pass.get(P::SceneUser); + let list=GpuRegion::new(list.gpu_va()?,list.size()).map_err(|_|EINVAL)?; + let metadata=pass.get(P::Scene); + let owner=Region::new(metadata.va(),metadata.size()).map_err(|_|EINVAL)?; + let mut bytes=[0;scene::EXTENDED_SIZE]; + scene::encode_extended(&mut bytes,owner,scratch_gpu,scratch_fw,list).map_err(|_|EINVAL)?; + self.passes[self.slot].get_mut(P::Scene).write(0,&bytes)?; + } else { let entry_offset = (self.draw % u64::from(MANAGER_SCENE_COUNT)) as usize * 4; let scene_list = &mut self.objects[s::BM_SCENES]; if entry_offset + 4 > scene_list.size() { return Err(ERANGE); } @@ -560,7 +661,8 @@ impl Render { let mut bytes = [0; scene::SIZE]; scene::Scene { manager_entry, manager_misc, user_buffer }.encode(&mut bytes).map_err(|_| EINVAL)?; self.passes[self.slot].get_mut(P::Scene).write(0, &bytes)?; - self.prepare_tiler(r,draw)?; + } + self.prepare_tiler(r,usc,draw)?; self.patch_context()?; self.prepare_fragment(r,usc,draw)?; self.prepare_sequences(draw)?; @@ -655,7 +757,13 @@ impl Render { if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value || self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head) || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} - } Ok(true) + } + if self.profile==super::m3_profile::Profile::Extended && self.draw!=0 { + for pool in UMA_POOLS { + if self.objects[pool.manager].read_u64(0x54)?!=self.draw {return Ok(false);} + } + } + Ok(true) } pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { // Retain allocator state as well as payloads: an exhausted parameter diff --git a/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs b/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs index f9a0483e77b21..3d421e9906692 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render_sequence.rs @@ -184,3 +184,171 @@ impl Fragment { Ok(if timed {LENGTH} else {without_timestamps(bytes,TIMESTAMP_START,FINALIZE,LENGTH,0xa4)}) } } + +impl Owners { + pub(crate) fn encode_extended(&self, fragment:bool, + auxiliary:super::m3_compute_layout::GpuVa, serial:u64, output:&mut[u8]) -> Result { + if self.context>=64 || serial==0 || self.stamp==0 {return Err(Error::Address);} + let command=self.command.at(0,if fragment {0xca0}else{0x940})?; + let sequence=self.sequence.at(0,STORAGE)?; + let activity=self.stats.at(0,8)?; + let scene_tail=self.scene.get().checked_add(0x54).ok_or(Error::Address)?; + let (clock,pair,aux_time,elapsed)=if fragment {(0xc08,0xc10,0xc70,0xb84)}else{(0x8c0,0x8c8,0x928,0x83c)}; + let first=if fragment {0x1e4}else{0x1cc}; + let finalize=if fragment {0x280}else{0x268}; + let pool=if fragment {0x1b0}else{0x190}; + let length=if fragment {0x370}else{0x300}; + let uuid=if fragment {fragment::UUID}else{ta::UUID}; + let bytes=output.get_mut(..STORAGE).ok_or(Error::Size)?; + bytes.fill(0); + word(bytes,0,if fragment {7}else{5}); + pointer(bytes,0x14,command+if fragment {0x80}else{0x40}); + pointer(bytes,pool,self.pool.get()); + pointer(bytes,pool+0x18,auxiliary.get()); + pointer(bytes,pool+0x28,serial); + word(bytes,pool+0x30,u32::from(fragment)); + if fragment { + pointer(bytes,0x1c,self.scene.get()); + pointer(bytes,0x24,activity); + pointer(bytes,0x2c,command+0xb70); + pointer(bytes,0x34,scene_tail); + pointer(bytes,0x3c,command+0xbb0); + pointer(bytes,0x44,command+0xbb4); + pointer(bytes,0x4c,self.queue.get()); + pointer(bytes,0x54,command); + word(bytes,0x5c,self.context); + word(bytes,0x60,1); + pointer(bytes,0x7c,command+0xa58); + pointer(bytes,0x84,command+0xbf0); + word(bytes,0xa0,uuid); + } else { + pointer(bytes,0x1c,self.manager.get()); + pointer(bytes,0x24,self.scene.get()); + pointer(bytes,0x2c,activity); + pointer(bytes,0x34,self.queue.get()); + pointer(bytes,0x3c,command+0x85c); + word(bytes,0x44,self.context); + word(bytes,0x48,1); + pointer(bytes,0x64,command+0x784); + pointer(bytes,0x6c,command+0x8a8); + word(bytes,0x7c,uuid); + } + for (at,opcode,selected) in [(first,0x8000_0003,pair),(first+0x50,3,pair+8)] { + word(bytes,at,opcode); + pointer(bytes,at+4,command+clock); + pointer(bytes,at+0xc,command+pair); + pointer(bytes,at+0x14,command+selected); + pointer(bytes,at+0x1c,self.queue.get()); + pointer(bytes,at+0x2c,command+aux_time); + pointer(bytes,at+0x34,command+elapsed); + pointer(bytes,at+0x44,uuid.into()); + } + word(bytes,first+0x4c,1); + word(bytes,finalize,if fragment {8}else{6}); + if fragment { + word(bytes,finalize+4,uuid); + pointer(bytes,finalize+0xc,self.fw_stamp.get()); + word(bytes,finalize+0x14,self.stamp); + pointer(bytes,finalize+0x1c,self.scene.get()); + pointer(bytes,finalize+0x24,self.manager.get()); + word(bytes,finalize+0x2c,1); + pointer(bytes,finalize+0x30,activity); + pointer(bytes,finalize+0x38,command+0xbb0); + pointer(bytes,finalize+0x40,command+0xbb4); + pointer(bytes,finalize+0x48,command+0xb70); + pointer(bytes,finalize+0x50,self.queue.get()); + pointer(bytes,finalize+0x58,command); + word(bytes,finalize+0x60,self.context); + pointer(bytes,finalize+0x64,command+0xa58); + pointer(bytes,finalize+0x98,sequence+pool as u64); + word(bytes,finalize+0xa0,(-(finalize as i32))as u32); + pointer(bytes,finalize+0xa5,command+0xc60); + word(bytes,finalize+0xad,1); + bytes[finalize+0xb1]=1; + // J613 terminates this record after the 16-bit event field. + bytes[finalize+0xb2..finalize+0xb4].copy_from_slice(&1u16.to_le_bytes()); + word(bytes,finalize+0xb4,0x4000_0002); + } else { + pointer(bytes,finalize+4,self.scene.get()); + pointer(bytes,finalize+0xc,self.manager.get()); + pointer(bytes,finalize+0x14,activity); + pointer(bytes,finalize+0x1c,self.queue.get()); + pointer(bytes,finalize+0x24,command+0x85c); + word(bytes,finalize+0x2c,self.context); + pointer(bytes,finalize+0x34,command+0x784); + word(bytes,finalize+0x40,uuid); + pointer(bytes,finalize+0x48,self.fw_stamp.get()); + word(bytes,finalize+0x50,self.stamp); + pointer(bytes,finalize+0x78,sequence+pool as u64); + word(bytes,finalize+0x80,(-(finalize as i32))as u32); + pointer(bytes,finalize+0x85,command+0x918); + bytes[finalize+0x8d]=1; + word(bytes,finalize+0x94,0x4000_0002); + } + Ok(length) + } +} + +#[cfg(test)] +mod native_tests { + use super::*; + #[test] fn j613_fragment_ends_before_the_m4_tail_and_keeps_relative_restart() { + let fw=|index:u64|FirmwareVa::new(0xffff_fc25_2000_0000+index*0x4000).unwrap(); + let o=Owners {command:Region::new(fw(0).get(),0xca0).unwrap(),sequence:Region::new(fw(1).get(),STORAGE).unwrap(), + stats:Region::new(fw(2).get(),0x1000).unwrap(),notifier:Region::new(fw(3).get(),0x200).unwrap(), + queue:fw(4),scene:fw(5),manager:fw(6),pool:fw(7),fw_stamp:fw(8),context:3,stamp:0x3d000100}; + let mut output=[0xa5;STORAGE+8]; + let length=o.encode_extended(true,super::super::m3_compute_layout::GpuVa::new(0x72_3000_0200).unwrap(),1,&mut output).unwrap(); + assert_eq!(length,0x370); + let word=|at|u32::from_le_bytes(output[at..at+4].try_into().unwrap()); + assert_eq!(word(0x1e4),0x8000_0003); + assert_eq!(word(0x230),1); + assert_eq!(word(0x234),3); + assert_eq!(word(0x280),8); + assert_eq!(word(0x320),(-0x280i32)as u32); + assert_eq!(word(0x334),0x4000_0002); + assert_eq!(&output[STORAGE..],&[0xa5;8]); + } +} + +/// Native start records reserve one packed attachment list per engine. The +/// matching finalizer flag enables completion visibility for that same list. +pub(crate) const fn extended_attachment_offsets(fragment:bool)->(usize,usize) { + if fragment {(0xa8,0x324)} else {(0x88,0x2ec)} +} + +/// Send both pieces through the same bounded writer used for firmware storage. +#[cfg(any(test, CONFIG_DRM_ASAHI_M3_T8122))] +pub(crate) fn patch_extended_attachments(fragment:bool, + attachments:&super::agx_attachments::Attachments, + mut write:impl FnMut(usize,&[u8])->Result<(),E>)->Result<(),E> { + let (start,flag)=extended_attachment_offsets(fragment); + write(start,&attachments.0)?; + write(flag,&[u8::from(attachments.count()!=0)]) +} + +#[cfg(test)] +mod attachment_tests { + use super::*; + use super::super::agx_attachments::{Attachments,SIZE}; + #[test] fn each_engine_keeps_its_list_and_can_clear_a_previous_list() { + for (fragment,start,flag,pool) in [(false,0x88,0x2ec,0x190),(true,0xa8,0x324,0x1b0)] { + let mut bytes=[0xa5;STORAGE]; + let list=Attachments::new(&[(0x100_0040,0x180),(0x200_0000,0x4000)]).unwrap(); + let mut apply=|list:&Attachments|patch_extended_attachments(fragment,list,|at,data|{ + bytes.get_mut(at..at+data.len()).ok_or(Error::Size)?.copy_from_slice(data);Ok::<(),Error>(()) + }); + apply(&list).unwrap(); + assert_eq!(&bytes[start..start+SIZE],&list.0); + assert_eq!(bytes[flag],1); + assert_eq!(&bytes[pool..pool+8],&[0xa5;8]); + patch_extended_attachments(fragment,&Attachments::EMPTY,|at,data|{ + bytes[at..at+data.len()].copy_from_slice(data);Ok::<(),Error>(()) + }).unwrap(); + assert!(bytes[start..start+SIZE].iter().all(|b|*b==0)); + assert_eq!(bytes[flag],0); + assert_eq!(bytes[start-1],0xa5); + assert_eq!(bytes[start+SIZE],0xa5); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 058c6610399ae..3b1f3d6d08535 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -390,7 +390,7 @@ impl Inner { if found.is_none() { let job=match control { crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), - crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.render_stats().map_err(early)?,self.device.soc().clusters,self.device.soc().registers,self.device.firmware().profile()).map_err(early)?), }; self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; } else { @@ -439,6 +439,7 @@ impl Inner { let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,chip,pair)); j.append(command,usc,coalesce,timed)?; j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.render_attachments[*next])?; coalesced_stamps+=u64::from(coalesce); } Ok(()) @@ -563,6 +564,7 @@ impl Inner { let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,self.device.soc().chip_id,pair)); j.append_after(command,usc,false,predecessor,timed)?; j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.render_attachments[*i])?; dependencies+=usize::from(predecessor.is_some()); previous=Some((true,j.completion_dependency()?)); }, diff --git a/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs b/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs index 880cf4c2b9a75..ea3c2064ef39e 100644 --- a/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_scene_layout.rs @@ -39,3 +39,43 @@ impl Scene { Ok(()) } } + +pub(crate) const EXTENDED_SIZE: usize = 0x400; + +/// Encode a native scene header from its retained aliases. The local +/// statistics and parameter scratch remain part of this same owned record. +pub(crate) fn encode_extended(output: &mut [u8], owner: super::m3_init_layout::Region, + scratch_gpu: super::m3_pool_layout::GpuRegion, scratch_fw: FirmwareVa, + scene_list: super::m3_pool_layout::GpuRegion) -> Result<(), Error> { + let statistics = owner.at(0x200, 0x40).map_err(|_| Error::Size)?; + owner.at(0, EXTENDED_SIZE).map_err(|_| Error::Size)?; + if scratch_gpu.size() < 0x4000 || scene_list.size() < 0x4000 { return Err(Error::Size); } + let compact = super::agx_render_state::compact(scene_list.base()).map_err(|_| Error::Address)?; + let bytes = output.get_mut(..EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + bytes[..8].copy_from_slice(&scratch_gpu.base().to_le_bytes()); + bytes[8..16].copy_from_slice(&scratch_fw.get().to_le_bytes()); + bytes[0x28..0x30].copy_from_slice(&compact.to_le_bytes()); + bytes[0x40..0x48].copy_from_slice(&statistics.to_le_bytes()); + Ok(()) +} + +#[cfg(test)] +mod extended_tests { + use super::*; + use super::super::{m3_init_layout::Region, m3_pool_layout::GpuRegion}; + #[test] fn native_scene_keeps_scratch_aliases_and_local_statistics_distinct() { + let owner = Region::new(0xffff_fc25_0010_0000, EXTENDED_SIZE).unwrap(); + let scratch = GpuRegion::new(0x10_0200_0000, 0x4000).unwrap(); + let scratch_fw = FirmwareVa::new(0xffff_fc25_0020_0000).unwrap(); + let list = GpuRegion::new(0x10_0300_0000, 0x4000).unwrap(); + let mut bytes = [0xa5; EXTENDED_SIZE + 8]; + encode_extended(&mut bytes, owner, scratch, scratch_fw, list).unwrap(); + let read = |at| u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()); + assert_eq!(read(0), scratch.base()); + assert_eq!(read(8), scratch_fw.get()); + assert_eq!(read(0x28), 0x300_0000); + assert_eq!(read(0x40), owner.at(0x200, 0x40).unwrap()); + assert_eq!(&bytes[EXTENDED_SIZE..], &[0xa5; 8]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs b/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs index 9b8d503836d0a..0a2262858556d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_shared_layout.rs @@ -100,3 +100,29 @@ pub(crate) fn fault_buffers()->impl Iterator { .chain((PB_FIRST..PB_FIRST+PB_GROUPS).map(|i|(i,256))) .chain([(TILEMAP_LABEL,1280),(HEAP_METADATA_LABEL,256)]) } + +pub(crate) fn allocation_for(profile: super::m3_profile::Profile, index: usize) -> Result { + let mut owner = allocation(index)?; + if profile == super::m3_profile::Profile::Legacy { return Ok(owner); } + if owner.space == Space::Firmware { + owner.address = 0xffff_fc24_0000_0000 + index as u64 * 0x10_0000; + } + match index { + // Extended firmware reads the uncached kernel aliases supplied to + // encode_extended; these lower client mappings are GPU-only. + PARAMETER_PAGE_LIST | PARAMETER_BLOCK_LIST => owner.access = Access::GpuCachedRw, + 8 | 20 => owner.size = 0x20000, + 9 | 21 => { owner.address &= !0x3fff; owner.size = 0x4000; } + 10 | 22 => { + owner.size = 0x4000; + owner.gpu_alias = Some(0x72_3000_0000 + u64::from(index == 22) * 0x4000); + } + GPU_CONTEXT => owner.size = 0x38, + NOTIFIER => owner.size = 0x400, + EVENT_COUNT => owner.size = 16, + BUFFER_MANAGER => owner.size = 0xc0, + FRAGMENT_QUEUE | TA_QUEUE => owner.size = 0x24c0, + _ => {} + } + Ok(owner) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 2aac0dfad36ff..29d2bc7de5f68 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -41,6 +41,7 @@ pub(crate) struct Packet { timestamps:KVec<[Option;4]>, pub(crate) attachments:KVec, pub(crate) wide_visibility:KVec, + pub(crate) render_attachments:KVec<[crate::m3::agx_attachments::Attachments;2]>, completion:UserFence, finish_claimed: AtomicBool, /// Set before a batch with any of this packet's commands can change firmware-visible state, @@ -196,7 +197,7 @@ impl queue::Queue for Queue { fn submit(&mut self,_reg_data:&crate::driver::AsahiRegData<'_>,_id:u64,mut syncs:KVec,in_sync_count:usize, raw:&[u8],objects:Pin<&xarray::XArray>>)->Result { let vm_job=self.vm.retain_job()?; - let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new(); + let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new();let mut render_attachments=KVec::new(); while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? { if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command { if let Err(e)=validate_render(payload,self.usc,&AddressSpace(&self.vm),self.early_tiling) { @@ -211,6 +212,7 @@ impl queue::Queue for Queue { Destination::resolve(objects,payload.fragment_timestamps.start)?, Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?; flushes.push(crate::m3::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; + render_attachments.push([encode_attachments(&self.vm,&vertex_attachments)?,encode_attachments(&self.vm,&fragment_attachments)?],GFP_KERNEL)?; visibility.push(false,GFP_KERNEL)?; continue; } @@ -226,6 +228,7 @@ impl queue::Queue for Queue { // The common translator validates the original command/attachment // addresses; this M3 scheduling hint changes none of them. payload.flags &= !wide; + render_attachments.push([crate::m3::agx_attachments::Attachments::EMPTY;2],GFP_KERNEL)?; flushes.push(encode_attachments(&self.vm,&attachments)?,GFP_KERNEL)?; let c=agx_uapi::translate_compute_command(payload,attachments, agx_uapi::QueueUscWindow{base:self.usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}, @@ -239,7 +242,7 @@ impl queue::Queue for Queue { commands.push(Command::Compute(crate::m3::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; } parser.finish().map_err(|_|EINVAL)?; - let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, + let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility,render_attachments, completion:self.fences.new_fence(0,Completion)?.into(), finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs b/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs index 92d533efa8234..2cf5e18914043 100644 --- a/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs +++ b/drivers/gpu/drm/asahi/m3/m3_sync_layout.rs @@ -91,3 +91,58 @@ impl Barrier { Ok(()) } } + +impl InitBufferManager { + /// The extended rebuild path requires a zero staged-block count. Appending + /// already-staged blocks would leave a previous hardware descriptor live. + pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> { + if self.context >= 64 || self.slot >= 32 { return Err(Error::Address); } + Self { context: self.context, slot: self.slot, block_count: 0, + manager: self.manager, stamp: self.stamp }.encode(output) + } +} + +impl Barrier { + pub(crate) const EXTENDED_SIZE: usize = 0x40; + + pub(crate) fn encode_extended(&self, output: &mut [u8]) -> Result<(), Error> { + if self.event >= 128 || self.wait_value == 0 || self.self_value == 0 { + return Err(Error::Address); + } + let bytes = output.get_mut(..Self::EXTENDED_SIZE).ok_or(Error::Size)?; + bytes.fill(0); + u32_at(bytes, 0, 4); + address_at(bytes, 4, self.stamp); + address_at(bytes, 12, self.stamp); + u32_at(bytes, 0x14, self.wait_value); + u32_at(bytes, 0x20, self.event); + u32_at(bytes, 0x24, self.self_value); + if matches!(self.dependency, Dependency::PreviousPass) { u32_at(bytes, 0x3c, 1); } + Ok(()) + } +} + +#[cfg(test)] +mod extended_tests { + use super::*; + #[test] fn full_rebuild_never_uses_the_legacy_staged_count() { + let mut bytes = [0xa5; INIT_BM_SIZE]; + InitBufferManager { context: 3, slot: 7, block_count: 2048, + manager: FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), stamp: 0x100 } + .encode_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[12..20], &[0; 8]); + } + #[test] fn extended_dependencies_use_their_native_tail_and_preserve_trailing_storage() { + for previous in [false, true] { + let mut bytes = [0xa5; Barrier::EXTENDED_SIZE + 8]; + Barrier { stamp: FirmwareVa::new(0xffff_fc24_0010_0000).unwrap(), wait_value: 0x100, + event: 2, self_value: 0x200, uuid: 9, + dependency: if previous { Dependency::PreviousPass } else { Dependency::TilerToFragment } } + .encode_extended(&mut bytes).unwrap(); + assert_eq!(&bytes[0x18..0x20], &[0; 8]); + assert_eq!(&bytes[0x20..0x24], &[2, 0, 0, 0]); + assert_eq!(u32::from_le_bytes(bytes[0x3c..0x40].try_into().unwrap()), previous.into()); + assert_eq!(&bytes[Barrier::EXTENDED_SIZE..], &[0xa5; 8]); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs b/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs index eeda857a3ed88..caa61621e0b04 100644 --- a/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs +++ b/drivers/gpu/drm/asahi/m3/m3_tiler_command.rs @@ -3,6 +3,17 @@ use super::{agx_render::Geometry, m3_compute_layout::{GpuVa,RegisterSet}, m3_init_layout::{Error,Region}, m3_queue_layout::FirmwareVa, m3_render_sequence::ta}; pub(crate) const SIZE:usize=0x92b; +pub(crate) const EXTENDED_SIZE:usize=0x940; +pub(crate) const EXTENDED_FLUSH:usize=0x894; +pub(crate) struct Dispatch26 { + pub(crate) incarnation:u8, + pub(crate) serial:u64, + pub(crate) peer_stamp:u32, + pub(crate) usc_window:u64, + pub(crate) firmware_aux:FirmwareVa, + pub(crate) scene_scratch:GpuVa, +} + pub(crate) const REGISTERS:usize=ta::REGISTERS; /// The registers G15G does not take; the others keep the G15S order. pub(crate) const G15G_OMIT:[u32;14]=[0x1c041,0x1c9d0,0x1c948,0x1c888,0x1c890,0x1c918,0x1c089,0x1c9e0,0x16c41,0x1ca40,0x1c9a8,0x1c920,0xa5a1,0xd419]; @@ -57,17 +68,32 @@ impl Command { /// All address checks precede mutation. Zero all reserved fields, then /// emit semantic registers and explicit packed command/tail fields. pub(crate) fn encode(&self,out:&mut[u8])->Result<(),Error> { - if out.len() Result<(),Error> { + self.encode_format(out, Some(dispatch)) + } + fn encode_format(&self, out: &mut [u8], dispatch: Option) -> Result<(),Error> { + let extended = dispatch.is_some(); + let size = if extended { EXTENDED_SIZE } else { SIZE }; + if out.len()= 1 << 42 || self.register_set != RegisterSet::G15G { + return Err(Error::Address); + } + } if self.context>=64 || self.page_count==0 || self.tpc_bytes==0 {return Err(Error::Address);} let registers=self.gpu_alias.get().checked_add(REGISTERS as u64).ok_or(Error::Address)?; - GpuVa::new(self.gpu_alias.get().checked_add((SIZE-3) as u64).ok_or(Error::Address)?) + GpuVa::new(self.gpu_alias.get().checked_add(((size-1)&!3) as u64).ok_or(Error::Address)?) .map_err(|_|Error::Address)?; - let seq=self.sequence.at(0,ta::LENGTH)?; + let sequence_size = if extended { 0x300 } else { ta::LENGTH }; + let seq=self.sequence.at(0,sequence_size)?; let tilemap=compact(self.tilemap.get())?; let tpc=compact(self.tpc.get())?; let heap=compact(self.heap.get())?; let preemption=[compact(self.preemption[0].get())?,compact(self.preemption[1].get())?,compact(self.preemption[2].get())?]; - let user=compact(self.scene_user.get().checked_add(0x40).ok_or(Error::Address)?)?; + let user=compact(self.scene_user.get().checked_add(if extended {0} else {0x40}).ok_or(Error::Address)?)?; let g=&self.geometry; let layers=g.layers; let layer_metadata=compact(self.heap.get().checked_add(if layers>1 {0x200}else{0}).ok_or(Error::Address)?)?; @@ -99,23 +125,67 @@ impl Command { (0x1c830,0),(0x1ca30,user),(0x16c39,user), (0x1c910,0xa0_00000000|(self.initial_scene_entry.get()&0xf_ffffffff)|1), ]; - let bytes=&mut out[..SIZE];bytes.fill(0); // TA opcode is zero - pointer(bytes,4,1); // qualified work-command counter remains one + let bytes=&mut out[..size];bytes.fill(0); // TA opcode is zero + pointer(bytes,4,dispatch.as_ref().map_or(1, |native| native.serial)); word(bytes,CONTEXT,self.context); - for (offset,value) in [(0x14,self.notifier.get()),(0x24,self.manager.get()), - (0x2c,self.scene.get()),(0x34,self.empty.get()),(0x740,registers), + for (offset,value) in [(if extended {0x10} else {0x14},self.notifier.get()), + (if extended {0x20} else {0x24},self.manager.get()), + (if extended {0x28} else {0x2c},self.scene.get()), + (if extended {0x30} else {0x34},self.empty.get()),(0x740,registers), (0x760,self.tpc.get()),(0x768,self.tpc_bytes),(0x770,seq)] {pointer(bytes,offset,value);} let g15g=self.register_set==RegisterSet::G15G; let mut count=0; - for (register,value) in values.into_iter().filter(|(r,_)| !g15g || !G15G_OMIT.contains(r)) { - let n=count;count+=1; - word(bytes,REGISTERS+n*REGISTER_STRIDE,register); - pointer(bytes,REGISTERS+n*REGISTER_STRIDE+4,value); + let mut emit = |register, value| { + word(bytes, REGISTERS + count * REGISTER_STRIDE, register); + pointer(bytes, REGISTERS + count * REGISTER_STRIDE + 4, value); + count += 1; + }; + for (register, mut value) in values { + if g15g && G15G_OMIT.contains(®ister) { continue; } + if extended && matches!(register, 0x12099 | 0x1c898) { continue; } + if let Some(native) = &dispatch { + if register == 0x101e1 { + emit(0xa309, 0); + emit(0x1c8e0, u64::MAX); + emit(0x1c8e8, u64::MAX); + emit(0x1c898, 0); + } + if register == 0x1c910 { + let address = native.scene_scratch.get(); + let escaped = (address & (1 << 42)) >> 3; + let bias = if escaped == 0 { 0x70_0000_0000 } else { 0 }; + value = ((address + bias) & 0x7f_ffff_fffe) | escaped | 1; + } + } + emit(register, value); + } + if let Some(native) = &dispatch { + emit(0xa5a1, 0x60_0040_0020); + emit(0xd419, 0x2_0000_0001); + for register in [0x1ca10, 0x14a1, 0xa349] { emit(register, ta::UUID.into()); } + emit(0x10061, native.usc_window); } bytes[0x748..0x74a].copy_from_slice(&(count as u16).to_le_bytes()); bytes[0x74a..0x74c].copy_from_slice(&((count*REGISTER_STRIDE) as u16).to_le_bytes()); - word(bytes,SEQUENCE_LENGTH,ta::LENGTH as u32); + word(bytes,SEQUENCE_LENGTH,sequence_size as u32); word(bytes,0x77c,1); // fragment event slot + if let Some(native) = dispatch { + word(bytes, STAMP_VALUE, native.peer_stamp); + pointer(bytes, 0x7a8, preemption[0]); + pointer(bytes, 0x878, self.stamp.get()); + pointer(bytes, 0x880, self.fw_stamp.get()); + word(bytes, 0x888, self.stamp_value); + word(bytes, EXTENDED_FLUSH, 1); + word(bytes, 0x898, ta::UUID); + pointer(bytes, 0x8c8, self.timestamps[0].get()); + pointer(bytes, 0x8d0, self.timestamps[1].get()); + pointer(bytes, 0x8d8, self.user_timestamps[0].get()); + pointer(bytes, 0x8e0, self.user_timestamps[1].get()); + pointer(bytes, 0x8fe, self.pool.get()); + bytes[0x906] = 1; + pointer(bytes, 0x90f, native.firmware_aux.get()); + bytes[0x917] = native.incarnation; + } else { word(bytes,STAMP_VALUE,self.stamp_value); pointer(bytes,ta::STRUCT3+0x24,preemption[0]); word(bytes,ta::STRUCT3+0xbc,0x67); // retained producer encoder_id @@ -132,6 +202,41 @@ impl Command { (0x47,self.scratch.get())] {pointer(bytes,ta::COMMAND_TAIL+offset,value);} bytes[ta::COMMAND_TAIL+0x36]=1; // unk_846 bytes[ta::COMMAND_TAIL+0x4f]=1; // unk_85f + } Ok(()) } } + +#[cfg(test)] +mod extended_tests { + use super::*; + fn command() -> Command { + let fw=|index:u64|FirmwareVa::new(0xffff_fc25_1000_0000+index*0x4000).unwrap(); + let gpu=|index:u64|GpuVa::new(0x10_0100_0000+index*0x4000).unwrap(); + Command {context:3,register_set:RegisterSet::G15G,notifier:fw(0),manager:fw(1),scene:fw(2),empty:fw(3), + gpu_alias:GpuVa::new(0x72_1000_0000).unwrap(),sequence:Region::new(fw(4).get(),0x1000).unwrap(), + pool:fw(5),scratch:fw(6),stamp:fw(7),fw_stamp:fw(8),stamp_value:0x7a000100, + timestamps:[fw(9),fw(10)],user_timestamps:[fw(11),fw(12)],preemption:[gpu(0),gpu(1),gpu(2)], + tilemap:gpu(3),tpc:gpu(4),tpc_bytes:0x4000,heap:gpu(5),scene_user:gpu(6), + initial_scene_entry:fw(13),geometry:Geometry::new(64,64,super::super::agx_render::Utile::Pixels32, + super::super::agx_render::Utile::Pixels32,1).unwrap(),page_count:128, + vdm:0x100000,multisample:0,ppp:0,process_empty_tiles:false} + } + #[test] fn native_tiler_keeps_peer_completion_and_usc_window_explicit() { + let value=command();let mut bytes=[0xa5;EXTENDED_SIZE+8]; + value.encode_extended(Dispatch26 {incarnation:9,serial:2,peer_stamp:0x3d000100, + usc_window:1<<32,firmware_aux:value.scratch, + scene_scratch:GpuVa::new(0x10_0200_0000).unwrap()},&mut bytes).unwrap(); + let word=|at|u32::from_le_bytes(bytes[at..at+4].try_into().unwrap()); + assert_eq!(word(0x780),0x3d000100); + assert_eq!(word(0x888),value.stamp_value); + assert_eq!(word(0x894),1); + assert_eq!(word(0x748)&0xffff,66); + assert_eq!(bytes[0x917],9); + assert_eq!(&bytes[EXTENDED_SIZE..],&[0xa5;8]); + let registers=&bytes[REGISTERS..REGISTERS+66*REGISTER_STRIDE]; + assert!(!registers.chunks_exact(12).any(|r|u32::from_le_bytes(r[..4].try_into().unwrap())==0x12099)); + let usc=registers.chunks_exact(12).find(|r|u32::from_le_bytes(r[..4].try_into().unwrap())==0x10061).unwrap(); + assert_eq!(u64::from_le_bytes(usc[4..].try_into().unwrap()),1<<32); + } +} diff --git a/drivers/gpu/drm/asahi/m3/t8122_admission.rs b/drivers/gpu/drm/asahi/m3/t8122_admission.rs index fdc41024899f1..fb280ebcde520 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_admission.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_admission.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Read-only T8122 identity admission. Firmware startup remains disabled. +//! Read-only T8122 identity and firmware-interface admission. use super::m3_profile::Profile; @@ -53,6 +53,28 @@ pub(crate) fn validate_identity_for( }) } +/// Firmware-interface selection is exact and does not change physical GPU encodings. +pub(crate) fn profile_for(compat:&[u32],j613:bool)->Option { + match compat { + [14,8,3]=>Some(Profile::Legacy), + [26,6,2] if j613=>Some(Profile::Extended), + _=>None, + } +} + +#[cfg(not(test))] +pub(crate) fn loaded_profile(pdev:&kernel::platform::Device>) + ->kernel::error::Result { + use kernel::{c_str,prelude::*}; + let node=pdev.as_ref().of_node().ok_or(ENODEV)?; + let compat:KVec=node.get_property(c_str!("apple,firmware-compat"))?; + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + let j613 = super::m3_board::board_is(b"apple,j613"); + #[cfg(not(CONFIG_DRM_ASAHI_M3_T8122))] + let j613 = false; + profile_for(&compat, j613).ok_or(ENODEV) +} + /// Shape of the voltage-sorted running-state tables: one entry per distinct millivolt, /// and the highest primary frequency in MHz. Voltages round up, matching `PwrConfig`. /// The first point is the off state. Both primary and secondary frequencies must increase @@ -108,7 +130,7 @@ pub(crate) fn opp_table_shape(opps: impl IntoIterator) -> Opt pub(crate) fn read_identity( pdev: &kernel::platform::Device>, ) -> kernel::error::Result { - read_identity_for(pdev, Profile::Legacy) + read_identity_for(pdev, loaded_profile(pdev)?) } #[cfg(not(test))] @@ -300,3 +322,17 @@ mod tests { assert!(validate_identity(&[14, 8, 3], 1, 1, G15G_C0, [0x7f, 0, 0]).is_none()); } } + +#[cfg(test)] +mod selection_tests { + use super::*; + #[test] fn extended_admission_requires_exact_firmware_and_j613() { + for board in [false,true] { + assert_eq!(profile_for(&[14,8,3],board),Some(Profile::Legacy)); + assert_eq!(profile_for(&[26,6,2],board),if board{Some(Profile::Extended)}else{None}); + for wrong in [&[][..],&[26,6][..],&[26,6,1][..],&[26,6,2,0][..],&[14,8,4][..]] { + assert_eq!(profile_for(wrong,board),None); + } + } + } +} diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index 458603366c954..91cbefd298fe2 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -46,3 +46,63 @@ mod m3_state_layout; #[path = "../../drivers/gpu/drm/asahi/m3/m3_compute_storage.rs"] mod m3_compute_storage; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_parameter_layout.rs"] +mod m3_parameter_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_sync_layout.rs"] +mod m3_sync_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_render.rs"] +mod agx_render; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_render_state.rs"] +mod agx_render_state; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_scene_layout.rs"] +mod m3_scene_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_command_layout.rs"] +mod m3_command_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_render_sequence.rs"] +mod m3_render_sequence; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_tiler_command.rs"] +mod m3_tiler_command; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_pass_layout.rs"] +mod m3_pass_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_shared_layout.rs"] +mod m3_shared_layout; + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_fragment_command.rs"] +mod m3_fragment_command; + +#[path = "../../drivers/gpu/drm/asahi/m3/agx_attachments.rs"] +mod agx_attachments; + +#[test] +fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() { + use m3_pass_layout::{Access, Space}; + use m3_profile::Profile; + use m3_shared_layout as shared; + for index in 0..shared::COUNT { + let original = shared::allocation(index).unwrap(); + let legacy = shared::allocation_for(Profile::Legacy, index).unwrap(); + assert_eq!(legacy, original); + let extended = shared::allocation_for(Profile::Extended, index).unwrap(); + if [shared::PARAMETER_PAGE_LIST, shared::PARAMETER_BLOCK_LIST].contains(&index) { + assert_eq!(extended.space, Space::ClientGpu); + assert_eq!(extended.address, original.address); + assert_eq!(extended.size, original.size); + assert_eq!(extended.access, Access::GpuCachedRw); + assert_eq!(extended.access.pte(), 0xc0000000000080); + assert_eq!(original.access, Access::GpuFirmwareCachedRw); + } + if extended.space == Space::ClientGpu { + assert_ne!(extended.access, Access::GpuFirmwareCachedRw); + } + } +} From efcd402f4ccd5ecfb91745a944c328486af3745e Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 16:28:32 -0500 Subject: [PATCH 23/37] drm/asahi: admit authorized HIGH queues on Extended T8122 Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/file.rs | 10 ++++- drivers/gpu/drm/asahi/m3/m3_drm.rs | 5 +++ tools/asahi/queue-priority-tests.py | 69 +++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 tools/asahi/queue-priority-tests.py diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 24e8db30e580e..a09fa33ed02a4 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1350,9 +1350,15 @@ impl File { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } - // HIGH has only been run on G17; G13 and G14 keep refusing it. + // Keep support tied to a backend and admitted profile. G13/G14 and + // unqualified M3 profiles retain their previous rejection. + let high_supported = reg_data.gpu.as_any().is::(); + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + let high_supported = high_supported || reg_data.gpu.as_any() + .downcast_ref::() + .is_some_and(|gpu| gpu.supports_high_priority()); if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !reg_data.gpu.as_any().is::() + && !high_supported { cls_pr_debug!( Errors, diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 8aedd920f31dc..555edf7074a6b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -96,6 +96,11 @@ pub(crate) struct Backend { } impl Backend { + /// HIGH promises host publication ordering, not firmware preemption. + pub(crate) fn supports_high_priority(&self) -> bool { + self.soc.chip_id == 0x8122 && self.profile == super::m3_profile::Profile::Extended + } + /// Admits and starts the M3 runtime on `soc` (the T8122), with the DRM device `drm` that the /// caller registers afterwards. pub(crate) fn start( diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py new file mode 100644 index 0000000000000..5ecce0620588d --- /dev/null +++ b/tools/asahi/queue-priority-tests.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the production queue admission gate with controlled credentials/backends.""" +from pathlib import Path +import subprocess +import tempfile +root = Path(__file__).resolve().parents[2] +s = (root / 'drivers/gpu/drm/asahi/file.rs').read_text() +a = s.index(' if data.flags != 0', s.index('pub(crate) fn queue_create(')) +b = s.index(' let queues_xa', a) +gate = s[a:b] +predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n }', 1)[0] +unit = r''' +#![allow(dead_code, non_upper_case_globals)] +use std::any::Any; +const EINVAL:i32=22; const EACCES:i32=13; +macro_rules! cls_pr_debug { ($($arg:tt)*) => {} } +mod uapi { + pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME:u32=3; + pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH:u32=2; +} +mod bindings { pub const CAP_SYS_NICE:u32=23; + pub unsafe fn capable(_:u32)->bool { CAP.with(|c|c.get()) } + thread_local! {pub static CAP:std::cell::Cell=const {std::cell::Cell::new(false)};} +} +mod g17 {pub struct Gpu;} +mod m3 {pub mod m3_drm { + #[derive(PartialEq)] pub enum Profile {Legacy, Extended} + pub struct Soc {pub chip_id:u32} + pub struct Backend {pub soc:Soc,pub profile:Profile} + impl Backend {pub fn supports_high_priority(&self)->bool { PREDICATE }} +}} +struct Gpu(Box); impl Gpu {fn as_any(&self)->&dyn Any {&*self.0}} +struct Reg {gpu:Gpu} +struct Data {flags:u32,priority:u32} +struct File(bool); impl File {fn is_current_master(&self)->bool {self.0}} +fn admit(reg_data:&Reg,data:&Data,file:&File)->Result<(),i32> { GATE Ok(()) } +fn main() { + use m3::m3_drm::*; + let mut n=0; + for backend in 0..5 { for cap in [false,true] { for master in [false,true] { + let gpu:Box=match backend { + 0=>Box::new(g17::Gpu), + 1=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Extended}), + 2=>Box::new(Backend{soc:Soc{chip_id:0x8122},profile:Profile::Legacy}), + 3=>Box::new(Backend{soc:Soc{chip_id:0x6030},profile:Profile::Extended}), + _=>Box::new(())}; + let reg=Reg{gpu:Gpu(gpu)}; + bindings::CAP.with(|c|c.set(cap)); + for flags in [0,1,u32::MAX] { for priority in [0,1,2,3,4,u32::MAX] { + let expected=if flags!=0 || priority>=3 {Err(EINVAL)} + else if priority!=2 {Ok(())} + else if backend!=0 && !(backend==1 && cfg!(CONFIG_DRM_ASAHI_M3_T8122)) {Err(EINVAL)} + else if cap||master {Ok(())} else {Err(EACCES)}; + assert_eq!(admit(®,&Data{flags,priority},&File(master)),expected, + "backend={backend} cap={cap} master={master} flags={flags} priority={priority}"); n+=1; + }} + }}} + println!("{n} actual admission cases passed"); +} +'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate) +with tempfile.TemporaryDirectory(prefix='queue-priority-') as d: + p = Path(d) / 'gate.rs' + p.write_text(unit) + for enabled in [False, True]: + subprocess.run(['rustc', '--edition=2021', *(['--cfg', 'CONFIG_DRM_ASAHI_M3_T8122'] if enabled else []), str(p), '-o', str(p.with_suffix(''))], check=True) + # Without M3 compiled only the G17 backend is admitted. + subprocess.run([str(p.with_suffix(''))], check=True) +print('M3 enabled/disabled gate compilation passed') From c1c0e23c6e4f50e03038a22a6cab7f8d72e39a54 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 17:25:00 -0500 Subject: [PATCH 24/37] drm/asahi: test the M3 host priority mapping alongside admission Signed-off-by: Satya Benson --- tools/asahi/queue-priority-tests.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py index 5ecce0620588d..cfbbe8f36d4e6 100644 --- a/tools/asahi/queue-priority-tests.py +++ b/tools/asahi/queue-priority-tests.py @@ -10,6 +10,10 @@ b = s.index(' let queues_xa', a) gate = s[a:b] predicate = (root / 'drivers/gpu/drm/asahi/m3/m3_drm.rs').read_text().split('pub(crate) fn supports_high_priority(&self) -> bool {', 1)[1].split('\n }', 1)[0] +submit = (root / 'drivers/gpu/drm/asahi/m3/m3_submit.rs').read_text() +ordering = submit.split(' let urgency=', 1)[1].split(' Ok(Self', 1)[0] +ordering = 'let urgency=' + ordering +assert 'uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority' in s unit = r''' #![allow(dead_code, non_upper_case_globals)] use std::any::Any; @@ -35,6 +39,13 @@ struct Data {flags:u32,priority:u32} struct File(bool); impl File {fn is_current_master(&self)->bool {self.0}} fn admit(reg_data:&Reg,data:&Data,file:&File)->Result<(),i32> { GATE Ok(()) } +mod sched { + #[derive(Debug,PartialEq)] pub enum Priority {Low,Normal,High,Kernel} +} +fn ordering(priority:u32)->Result<(u8,sched::Priority),i32> { + ORDERING + Ok((urgency,priority)) +} fn main() { use m3::m3_drm::*; let mut n=0; @@ -56,9 +67,13 @@ "backend={backend} cap={cap} master={master} flags={flags} priority={priority}"); n+=1; }} }}} - println!("{n} actual admission cases passed"); + for (uapi,expected) in [(0,sched::Priority::Low),(1,sched::Priority::Normal),(2,sched::Priority::High)] { + assert_eq!(ordering(3-uapi),Ok((uapi as u8,expected))); + } + assert_eq!(ordering(4),Err(EINVAL)); + println!("{n} actual admission cases and LOW/MEDIUM/HIGH host mappings passed"); } -'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate) +'''.replace('PREDICATE', predicate.replace('super::m3_profile::Profile', 'Profile')).replace('GATE', gate).replace('ORDERING', ordering) with tempfile.TemporaryDirectory(prefix='queue-priority-') as d: p = Path(d) / 'gate.rs' p.write_text(unit) From ba872b99a0039f15a82fa0f91ed7b8ab4e5905c1 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 20:36:32 -0500 Subject: [PATCH 25/37] ci: register the M3 queue priority admission tests Signed-off-by: Satya Benson --- tools/asahi/run-host-tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 5dfda83574ea6..4ebce94ccb47d 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -53,6 +53,7 @@ def mutations(flag, names, base, expect=KILL): # test file -> [(arguments, expectation)]. '{out}' is a fresh directory. ASAHI_TESTS = { + 'queue-priority-tests.py': [([], PASS)], 'test-m3-power.py': [([], PASS)], 'test-m3-uapi.py': [([], PASS)], 'test-m3-mailbox.py': [([], PASS)], From bd92c01562acb283b2138ba90843606a4b924164 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 17:39:31 -0500 Subject: [PATCH 26/37] drm/asahi: back off interrupt-assisted M3 completion waits Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_completion.rs | 26 +++---- .../gpu/drm/asahi/m3/m3_completion_policy.rs | 77 +++++++++++++++++++ drivers/gpu/drm/asahi/m3/m3_drm.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_rtkit.rs | 14 ++-- drivers/gpu/drm/asahi/m3/mod.rs | 2 + tools/asahi/m3-fence-latency.py | 63 +++++++++++++++ tools/asahi/m3-runtime-tests.rs | 3 + 7 files changed, 166 insertions(+), 21 deletions(-) create mode 100644 drivers/gpu/drm/asahi/m3/m3_completion_policy.rs create mode 100644 tools/asahi/m3-fence-latency.py diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs index e06fcded9047e..387aaeca1b6c6 100644 --- a/drivers/gpu/drm/asahi/m3/m3_completion.rs +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -24,10 +24,9 @@ impl WorkItem for Completion { type Pointer = Arc; fn run(owner: Arc) { use crate::m3::m3_runtime::Service; - // Snapshots after a notification before sleeping for the next one: the completion - // stamps can trail the firmware's event message. - const TRAILING_POLLS: u32 = 4; - let (mut trailing, mut waited_at) = (0, u64::MAX); + // No deliberate spin: fsleep(5) selects udelay, not a scheduler sleep. + // Check immediately on IRQ, with backoff for trailing coherent witnesses. + let mut policy = super::m3_completion_policy::WaitPolicy::new(); loop { let (outcome, events) = { let mut guard = owner.shared.lock(); @@ -37,23 +36,20 @@ impl WorkItem for Completion { // Preserve the root-only CPU-overlap diagnostic: disabled waits retain // the runtime lock so VM/timestamp preparation cannot overlap the GPU. if let Service::Waiting(messages)=outcome { - if !events.cpu_overlap() {events.wait_past(messages);continue;} + if !events.cpu_overlap() { + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); + continue; + } } (outcome,events) }; match outcome { Service::Idle => break, - Service::Progress => { trailing = 0; waited_at = u64::MAX; } + Service::Progress => policy.progress(), Service::Waiting(messages) => { - if messages != waited_at { waited_at = messages; trailing = 0; } - if trailing < TRAILING_POLLS { - trailing += 1; - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5)); - } else { - // Until the next firmware event, or a bounded 100 us resnapshot. - events.wait_past(messages); - trailing = 0; - } + // IRQs and timer expiry alone are not retirement progress. + // Keep backing off until service publishes or retires work. + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); } } } diff --git a/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs new file mode 100644 index 0000000000000..9c0da8c3246e3 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded fallback for witnesses which can become visible after the last IRQ. +//! +//! Notifications request a fresh snapshot, but do not prove witness progress. +//! Only publication or retirement resets this policy. In particular, neither a +//! timer wake nor a stream of early firmware notifications renews a spin budget. + +pub(crate) struct WaitPolicy { + next_us: u64, +} + +impl WaitPolicy { + pub(crate) const fn new() -> Self { + Self { next_us: 25 } + } + + pub(crate) fn progress(&mut self) { + self.next_us = 25; + } + + /// No deliberate busy delay: IRQs still wake each sleep immediately. + /// The caller supplies the root-selected ceiling (25..=1000 us). + pub(crate) fn next_wait_us(&mut self, ceiling_us: u64) -> u64 { + let wait = self.next_us.min(ceiling_us.clamp(25, 1000)); + self.next_us = (wait * 2).min(1000); + wait + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_m3_completion_policy))] +mod tests { + use super::WaitPolicy; + + #[test] + fn no_progress_backs_off_and_stays_bounded() { + let mut policy = WaitPolicy::new(); + for expected in [25, 50, 100, 200, 400, 800, 1000, 1000] { + assert_eq!(policy.next_wait_us(1000), expected); + } + for _ in 0..100_000 { + assert_eq!(policy.next_wait_us(1000), 1000); + } + } + + #[test] + fn publication_or_retirement_restarts_fast_detection() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + policy.progress(); + assert_eq!(policy.next_wait_us(1000), 25); + assert_eq!(policy.next_wait_us(1000), 50); + } + + #[test] + fn live_ceiling_changes_apply_to_the_next_wait() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + assert_eq!(policy.next_wait_us(25), 25); + assert_eq!(policy.next_wait_us(1000), 50); + assert_eq!(policy.next_wait_us(100), 100); + assert_eq!(policy.next_wait_us(100), 100); + } + + #[test] + fn fallback_keeps_timeout_checks_frequent_without_any_irq() { + let mut policy = WaitPolicy::new(); + let mut elapsed_us = 0; + while elapsed_us < 2_000_000 { + let wait = policy.next_wait_us(1000); + assert!(wait <= 1000); + elapsed_us += wait; + } + // This bounds the requested delay, not scheduler dispatch latency. + assert!(elapsed_us < 2_001_000); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 555edf7074a6b..04759396c7757 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -38,7 +38,7 @@ struct CompletionView(Arc); impl kernel::debugfs::Writer for CompletionView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); - writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", + writeln!(f, "version=2 policy=adaptive wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", us, irqs, sleeps, timers, preparations) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs index be81c4a4e9d92..ba8ad365ce20a 100644 --- a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -38,7 +38,7 @@ impl EventWait { Arc::pin_init( pin_init!(EventWait { count: AtomicU64::new(0), - wait_us: AtomicU64::new(100), + wait_us: AtomicU64::new(1000), overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()), cpu_preparations: AtomicU64::new(0), sleeps: AtomicU64::new(0), @@ -55,7 +55,7 @@ impl EventWait { self.arrived.complete(); } - /// Diagnostic tuning changes only the bounded resnapshot cadence, never + /// Diagnostic tuning sets the adaptive fallback ceiling in microseconds, never /// stamps, pipe retirement, event counts, fences or the two-second timeout. /// Atomic publication permits root to compare policies in the same boot. pub(crate) fn set_wait_us(&self, us: u64) -> Result { @@ -64,6 +64,10 @@ impl EventWait { Ok(()) } + pub(crate) fn wait_ceiling_us(&self) -> u64 { + self.wait_us.load(Ordering::Relaxed) + } + pub(crate) fn cpu_overlap(&self) -> bool { self.overlap_cpu.load(Ordering::Relaxed) } @@ -82,11 +86,11 @@ impl EventWait { } /// Sleep until a notification later than `seen` has been recorded or - /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250, + /// the selected 25..1000 us fallback elapses. A one-jiffy fallback alone takes up to 4 ms at HZ=250, /// even when the GPU stamp arrives just after its notification. The timer /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting. /// Only the single completion worker may call this method. - pub(crate) fn wait_past(self: &Arc, seen: u64) -> bool { + pub(crate) fn wait_past(self: &Arc, seen: u64, wait_us: u64) -> bool { // Events can arrive while the worker is polling. Discard stale wakeup // tokens before checking the sequence. A concurrent event either moves // the sequence or leaves a token, including between the check and wait. @@ -98,7 +102,7 @@ impl EventWait { // on every return, before a subsequent wait can arm the timer. self.sleeps.fetch_add(1, Ordering::Relaxed); let timer = self.clone().start(Delta::from_micros( - self.wait_us.load(Ordering::Relaxed) as i64)); + wait_us.clamp(25, 1000) as i64)); // Retain one tick as a backup bound; normally either the real // interrupt or our high-resolution timer supplies the token. self.arrived.wait_for_completion_timeout(1); diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index d885da1a8647c..eafd5732bceac 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -133,6 +133,8 @@ mod t8122_knobs; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod t8122_start; +mod m3_completion_policy; + mod m3_compute_layout; mod m3_compute_sequence; diff --git a/tools/asahi/m3-fence-latency.py b/tools/asahi/m3-fence-latency.py new file mode 100644 index 0000000000000..c8428d65dbf0d --- /dev/null +++ b/tools/asahi/m3-fence-latency.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Join existing M3 timing logs. Input must be one boot, captured without gaps.""" +import collections +import json +import math +import re +import sys + +milestones = {} +packets = collections.defaultdict(list) +fences = [] +invalid = collections.Counter() +for line in open(sys.argv[1]): + fields = {k: int(v, 0) for k, v in re.findall(r'(\w+)=(-?0x[0-9a-fA-F]+|-?\d+)', line)} + if 'M3_BATCH_MILESTONE ' in line: + key = (fields['vm'], fields['publication_tick']) + if key in milestones: + invalid['duplicate_milestone'] += 1 + milestones[key] = fields + elif 'M3_BATCH_PACKET ' in line: + packets[(fields['vm'], fields['packet'])].append((fields['publication_tick'], fields['command'])) + elif 'M3_PACKET_FENCE ' in line: + fences.append(fields) +values = [] +previous_signal = {} +for f in sorted(fences, key=lambda f: f['signal_before_tick']): + identity = (f['vm'], f['packet']) + previous = previous_signal.get(identity, 0) + previous_signal[identity] = f['signal_after_tick'] + if f['status'] != 0: + invalid['error_fence'] += 1 + continue + linked = packets.get((f['vm'], f['packet']), []) + # Pointer reuse is disambiguated by the completed fence's physical counter. + candidates = [(tick, command) for tick, command in linked if previous < tick <= f['signal_before_tick'] and command == f['commands'] - 1] + if not candidates: + invalid['missing_final_command'] += 1 + continue + final_tick, _ = max(candidates) + batch = milestones.get((f['vm'], final_tick)) + if not batch or batch['mixed'] or batch['render_end_tick'] <= 0: + invalid['missing_or_non_render_end'] += 1 + continue + end = batch['render_end_tick'] + before, after = f['signal_before_tick'], f['signal_after_tick'] + if not (final_tick <= end <= before <= after): + invalid['invalid_counter_order'] += 1 + continue + # GPU execution end is earlier than full coherent retirement; this includes + # trailing witnesses and dispatch/retirement, rather than claiming an IRQ is done. + values.append(((before - end) / 24, (after - end) / 24)) + +def percentiles(data): + ordered = sorted(data) + return {str(p): ordered[max(0, math.ceil(len(ordered) * p / 100) - 1)] for p in (50, 95, 99, 100)} if ordered else {} + +print(json.dumps({'render_fences': len(values), 'invalid': invalid, + 'gpu_end_to_signal_before_us': percentiles([a for a, b in values]), + 'gpu_end_to_signal_after_us': percentiles([b for a, b in values]), + 'note': '24 MHz physical counter; interval brackets signal(); includes trailing witnesses. Not presentation latency.'}, indent=2)) +if not values: + sys.exit('No valid render fences: do not report latency from this capture') diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index 91cbefd298fe2..0d6d4e5841194 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -106,3 +106,6 @@ fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() { } } } + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_completion_policy.rs"] +mod m3_completion_policy; From 9f3b31006dd1ac17b751b73d9feceba550675f48 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Tue, 3 Nov 2026 17:59:21 -0500 Subject: [PATCH 27/37] drm/asahi: omit no-op M3 module pins without unload support Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 3b1f3d6d08535..8e25e2c3c602f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -1079,6 +1079,8 @@ impl Runtime { crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { + // Without module unloading, the C helper is an unbound no-op. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; core::mem::forget(transport);core::mem::forget(state); core::mem::forget(adopted); @@ -1348,6 +1350,7 @@ impl Drop for Runtime { for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { // SAFETY: a live module reference pins callbacks with retained DMA. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; return; } From 2d76bcead7cc87a1d487551ca4c25bd65f1a8055 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:22:54 -0500 Subject: [PATCH 28/37] drm/asahi: adapt the carried T8122 foundation to Aurora 7.3 Retain the newer G16 frontend and binding lifetime, and compose the public T8122 runtime with Aurora. Original dependency patches and their complete metadata are retained in the preceding carried commits. Signed-off-by: Satya Benson --- arch/arm64/configs/asahi.config | 35 +- drivers/gpu/drm/asahi/Kconfig | 39 +- drivers/gpu/drm/asahi/debug.rs | 2 +- drivers/gpu/drm/asahi/driver.rs | 259 ++++-- drivers/gpu/drm/asahi/file.rs | 273 +++++- drivers/gpu/drm/asahi/float.rs | 9 +- drivers/gpu/drm/asahi/g15/mod.rs | 71 +- drivers/gpu/drm/asahi/g15/uat.rs | 8 +- drivers/gpu/drm/asahi/gpu.rs | 123 ++- drivers/gpu/drm/asahi/mmu.rs | 1348 ++++++++++++++++++++++++------ rust/kernel/soc/apple/rtkit.rs | 90 +- 11 files changed, 1787 insertions(+), 470 deletions(-) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index ffecb85c3beb6..2f9a58855b04b 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -8,9 +8,16 @@ CONFIG_ARCH_APPLE=y # CONFIG_ARM64_4K_PAGES is not set CONFIG_ARM64_16K_PAGES=y # CONFIG_ARM64_64K_PAGES is not set +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y CONFIG_ARM64_MEMORY_MODEL_CONTROL=y +# 16K pages make a page block 32 MiB, and khugepaged then sets min_free_kbytes +# to 352 MiB on an 8 GB Mac. Keep page blocks at 2 MiB as on 4K kernels. +CONFIG_PAGE_BLOCK_MAX_ORDER=7 CONFIG_ARM_APPLE_CPUIDLE=y CONFIG_ARM_APPLE_SOC_CPUFREQ=m +CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y +# CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE is not set CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m @@ -20,7 +27,7 @@ CONFIG_NVME_APPLE=m CONFIG_MMC_SDHCI_PCI=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y -CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_PCIE_APPLE_T8140_BOOTSTRAP=y CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y CONFIG_IGB=m @@ -31,7 +38,7 @@ CONFIG_INPUT_MACSMC_INPUT=m CONFIG_I2C_APPLE=m CONFIG_SPI_APPLE=m CONFIG_SPI_APPLE_QSPI=m -CONFIG_SPMI_APPLE=m +CONFIG_SPMI_APPLE=y CONFIG_PINCTRL_APPLE_GPIO=m CONFIG_GPIO_MACSMC=m CONFIG_POWER_RESET_MACSMC=m @@ -41,21 +48,30 @@ CONFIG_SENSORS_APPLE_PMP_J613=m CONFIG_THERMAL_NETLINK=y CONFIG_NVME_HWMON=y CONFIG_APPLE_WATCHDOG=y +# m1n1 hands the ADT and its own log over as phram regions (/dev/mtd*). +CONFIG_MTD=y +CONFIG_MTD_PHRAM=m +# Apple NVRAM partitions are 4 KiB aligned (J700: 0x24000). +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y CONFIG_VIDEO_APPLE_AVD=m +CONFIG_VIDEO_APPLE_AVE=m CONFIG_VIDEO_APPLE_ISP=m +CONFIG_VIDEO_APPLE_PRORES=m +CONFIG_VIDEO_APPLE_JPEG=m CONFIG_DRM=y CONFIG_DRM_ASAHI=m +CONFIG_DRM_ASAHI_G16=y CONFIG_DRM_ASAHI_M3_T8122=y CONFIG_DRM_ASAHI_G15=y CONFIG_DRM_ADP=m CONFIG_DRM_APPLE=m +CONFIG_DRM_APPLE_DCP_T8132=y CONFIG_DRM_APPLE_AUDIO=y CONFIG_DRM_ACCEL=y CONFIG_DRM_ACCEL_ANE=m CONFIG_DRM_SIMPLEDRM=y CONFIG_SYSFB_SIMPLEFB=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m -CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_MCA=m CONFIG_SND_SOC_APPLE_MACAUDIO=m CONFIG_SND_SOC_CS42L83=m @@ -77,14 +93,14 @@ CONFIG_USB4=m CONFIG_USB4_APPLE_SOC=m CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m -CONFIG_RTC_DRV_MACSMC=m +CONFIG_RTC_DRV_MACSMC=y CONFIG_RTC_DRV_APPLE_PMU=m CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m -CONFIG_APPLE_SIO_T6030=y -CONFIG_MFD_MACSMC=m +CONFIG_MFD_MACSMC=y CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=y +CONFIG_APPLE_DART_J713_DCP=y CONFIG_APPLE_DOCKCHANNEL=m CONFIG_SERIAL_APPLE_DOCKCHANNEL=y CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y @@ -104,16 +120,20 @@ CONFIG_APPLE_SEP=m CONFIG_APPLE_PMGR_PWRSTATE=y CONFIG_APPLE_PMP=m CONFIG_APPLE_PMP_REPORT=y +CONFIG_APPLE_PMP_THERMAL=m +CONFIG_APPLE_T8132_PLATFORM_HANDOFF=y CONFIG_IIO_AOP_SENSOR_LAS=m CONFIG_IIO_AOP_SENSOR_ALS=m +CONFIG_IIO_AOP_SENSOR_MOTION=m CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y CONFIG_PWM_APPLE=m CONFIG_APPLE_AIC=y CONFIG_PHY_APPLE_ATC=m +CONFIG_PHY_APPLE_T6040_USB2=m CONFIG_PHY_APPLE_DPTX=m CONFIG_APPLE_M1_CPU_PMU=y CONFIG_NVMEM_APPLE_EFUSES=m -CONFIG_NVMEM_APPLE_SPMI=m +CONFIG_NVMEM_APPLE_SPMI=y CONFIG_MUX_APPLE_DPXBAR=m CONFIG_TYPEC_TPS6598X=m CONFIG_RESET_APPLE_CIO=y @@ -125,3 +145,4 @@ CONFIG_RDMA_SIW=m CONFIG_DMABUF_HEAPS=y CONFIG_DMABUF_HEAPS_SYSTEM=y CONFIG_UDMABUF=y +CONFIG_DETECT_HUNG_TASK=y diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 45e907b5af86d..109e1985ce559 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -9,7 +9,7 @@ config RUST_DRM_GPUVM select DRM_GPUVM config DRM_ASAHI - tristate "Asahi (DRM support for Apple AGX GPUs)" + tristate "Aurora (DRM support for Apple AGX GPUs)" depends on RUST depends on DRM=y depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) @@ -21,8 +21,27 @@ config DRM_ASAHI select RUST_DRM_GEM_SHMEM_HELPER select RUST_DRM_GPUVM select RUST_APPLE_RTKIT + select CRYPTO_LIB_SHA256 + help + DRM driver for Apple AGX GPUs, from the M1 family onwards. + + To compile this driver as a module, choose M here: the module will + be called aurora. It keeps "asahi" as a module alias. + +config DRM_ASAHI_G16 + bool "Apple M4 (G16G) support (experimental)" + depends on DRM_ASAHI && ARM64 + depends on APPLE_PMP=y || APPLE_PMP=DRM_ASAHI + depends on APPLE_CLPC=y || APPLE_CLPC=DRM_ASAHI || APPLE_CLPC=n + depends on THERMAL_OF help - DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) + Rust backend for the qualified J713/T8132 GPU firmware. + Load aurora explicitly after the native display and GFX TVM supplier + are ready. Firmware-visible storage remains retained until a verified + retirement or reboot. Keep the working kernel available during bring-up. + A rendering VM pins about 0.7 GiB; memory-cgroup limits are advisable. + An aborted receipt disables that command class until reboot. + Suspend, hibernation and kexec are blocked while native graphics owns DMA. config DRM_ASAHI_DEBUG_ALLOCATOR bool "Use debug allocator" @@ -36,21 +55,11 @@ config DRM_ASAHI_DEBUG_ALLOCATOR Say N unless you are debugging firmware structures or porting to a new firmware version. -config DRM_ASAHI_UAT_ADOPTION - bool - help - The firmware UAT handoff adoption shared by the G15 backend and the - T8122 runtime (g15/uat.rs, g15/regs.rs). - config DRM_ASAHI_M3_T8122 bool "M3 MacBook Air GPU runtime (T8122, 14.8.3 firmware)" depends on DRM_ASAHI - # The runtime adopts the firmware's UAT as the G15 backend does, without - # needing that backend: M3 Pro/Max keep their own setting. - # Its PMP votes call apple-pmp directly (m3_device.rs). - depends on APPLE_PMP=y || (APPLE_PMP=m && DRM_ASAHI=m) - select DRM_ASAHI_UAT_ADOPTION - select APPLE_T8122_GPU_PMP_LINK + # The runtime adopts the firmware's UAT through the G15 backend's handoff helpers. + depends on DRM_ASAHI_G15 select CRYPTO_LIB_SHA256 default n help @@ -68,8 +77,6 @@ config DRM_ASAHI_G15 depends on DRM_APPLE=y || (DRM_APPLE=m && DRM_ASAHI=m) select APPLE_T6030_GPU_PMP_LINK select CRYPTO_LIB_SHA256 - select RUST_FW_LOADER_ABSTRACTIONS - select DRM_ASAHI_UAT_ADOPTION default n help Enable the native G15 backend for explicitly admitted M3 Pro/Max diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs index e8087442d5320..6161d8d8dd748 100644 --- a/drivers/gpu/drm/asahi/debug.rs +++ b/drivers/gpu/drm/asahi/debug.rs @@ -91,7 +91,7 @@ pub(crate) enum DebugFlags { /// Update the cached global debug flags from the module parameter pub(crate) fn update_debug_flags() { - let flags = *module_parameters::debug_flags.value(); + let flags = module_parameters::debug_flags.value(); DEBUG_FLAGS.store(flags, Ordering::Relaxed); } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 43fa17412815d..6227bb4bf815b 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,11 +45,13 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { + binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, - /// The M3 T8122 runtime, stopped on unbind like the G17 GPU. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) g16: Option>, m3: Option, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. @@ -83,12 +85,18 @@ pub(crate) type AsahiDevRef = ARef; pub(crate) type AsahiRegData<'drm> = AsahiDrmRegistrationData<'drm>; /// DRM Driver metadata +/// +/// The major version names the UAPI generation in include/uapi/drm/asahi_drm.h +/// and only changes if that ABI ever breaks. The minor version goes up with +/// each new ioctl, flag or feature bit, so userspace can tell what a kernel +/// accepts without probing. The patch level marks fixes userspace may need to +/// detect. const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { - major: 0, + major: 1, minor: 0, patchlevel: 0, - name: c"asahi", - desc: c"Apple AGX Graphics", + name: c"aurora", + desc: c"Aurora GPU for Apple AGX", }; /// DRM Driver implementation for `AsahiDriver`. @@ -103,7 +111,6 @@ impl drm::driver::Driver for AsahiDrmDriver { type ParentDevice = platform::Device; const INFO: drm::driver::DriverInfo = INFO; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEAT_RENDER: bool = true; const FEAT_SYNCOBJ: bool = true; const FEAT_SYNCOBJ_TIMELINE: bool = true; @@ -143,68 +150,82 @@ pub(crate) enum ProbeConfig { M3(crate::m3::Chip), } -// OF Device ID table. -kernel::of_device_table!( - OF_TABLE, - MODULE_OF_TABLE, - ::IdInfo, - [ - ( - of::DeviceId::new(c"apple,agx-t8140"), - ProbeConfig::G17(&hw::t8140::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8103"), - ProbeConfig::Legacy(&hw::t8103::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8112"), - ProbeConfig::Legacy(&hw::t8112::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6000"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) - ), - ( - of::DeviceId::new(c"apple,agx-t6001"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) - ), - ( - of::DeviceId::new(c"apple,agx-t6002"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) - ), - ( - of::DeviceId::new(c"apple,agx-t6020"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) - ), - ( - of::DeviceId::new(c"apple,agx-t6021"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) - ), - ( - of::DeviceId::new(c"apple,agx-t6022"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) - ), - ( - of::DeviceId::new(c"apple,agx-t8122"), - ProbeConfig::M3(crate::m3::Chip::T8122) - ), - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - ( - of::DeviceId::new(c"apple,agx-t6030"), - ProbeConfig::M3(crate::m3::Chip::T6030) - ), - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - ( - of::DeviceId::new(c"apple,agx-t6031"), - ProbeConfig::M3(crate::m3::Chip::T6031) - ), - ( - of::DeviceId::new(c"apple,agx-t6032"), - ProbeConfig::M3(crate::m3::Chip::T6032) - ), - ] +// OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on +// its entries, so the entries that the native G15 backend owns when it is +// built are passed in by configuration. +macro_rules! asahi_of_device_table { + ($($g15_owned:tt)*) => { + kernel::of_device_table!( + OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c"apple,agx-t8140"), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6040"), + ProbeConfig::G17(&hw::t6040::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8103"), + ProbeConfig::Legacy(&hw::t8103::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8112"), + ProbeConfig::Legacy(&hw::t8112::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6000"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) + ), + ( + of::DeviceId::new(c"apple,agx-t6001"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) + ), + ( + of::DeviceId::new(c"apple,agx-t6002"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) + ), + ( + of::DeviceId::new(c"apple,agx-t6020"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) + ), + ( + of::DeviceId::new(c"apple,agx-t6021"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) + ), + ( + of::DeviceId::new(c"apple,agx-t6022"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) + ), + ( + of::DeviceId::new(c"apple,agx-t8122"), + ProbeConfig::M3(crate::m3::Chip::T8122) + ), + $($g15_owned)* + ( + of::DeviceId::new(c"apple,agx-t6032"), + ProbeConfig::M3(crate::m3::Chip::T6032) + ), + ] + ); + }; +} + +#[cfg(not(CONFIG_DRM_ASAHI_G15))] +asahi_of_device_table!( + ( + of::DeviceId::new(c"apple,agx-t6030"), + ProbeConfig::M3(crate::m3::Chip::T6030) + ), + ( + of::DeviceId::new(c"apple,agx-t6031"), + ProbeConfig::M3(crate::m3::Chip::T6031) + ), ); +#[cfg(CONFIG_DRM_ASAHI_G15)] +asahi_of_device_table!(); /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { @@ -212,19 +233,16 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>( - _pdev: &'bound platform::Device>, - this: Pin<&Self::Data<'bound>>, - ) { + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] if let Some(m3) = this.m3.as_ref() { - // As for G17: the runtime stops its coprocessor through device resources. m3.stop(); } + this.close_binding(); } /// Device probe function. @@ -233,6 +251,7 @@ impl platform::Driver for AsahiPlatformDriver { info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { debug::update_debug_flags(); + crate::memacct::LEDGER.configure(); dev_info!(pdev.as_ref(), "Probing...\n"); @@ -244,6 +263,7 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; + // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -260,6 +280,9 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -291,12 +314,20 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); + let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + // The DRM core created this card's debugfs directory when the device was registered; + // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and + // waits for its readers, before the registration data that owns the snapshot is dropped. + if let Some(snapshot) = debug_stats.as_ref() { + crate::stats_raw::register(reg.device(), snapshot); + } + // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); @@ -304,14 +335,17 @@ impl platform::Driver for AsahiPlatformDriver { let raw_stats = crate::sysfs_exports::register( raw_dev, snapshot, - stats.is_some() && *module_parameters::stats_export.value() != 0, + stats.is_some() && module_parameters::stats_export.value() != 0, )?; let driver = AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, raw_dev, raw_stats, stats, @@ -331,6 +365,9 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -340,10 +377,13 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -362,6 +402,7 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?; let gpu = backend.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); @@ -376,6 +417,9 @@ impl AsahiPlatformDriver { }; dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); Ok(AsahiPlatformDriverData { + binding, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, _device: reg.device().into(), _reg: reg, g17: None, @@ -395,6 +439,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -410,20 +457,75 @@ impl AsahiPlatformDriver { return Err(error); } }; - // G17's firmware channel format has no supported stats decoder yet. + let stats = gpu::Gpu::stats_snapshot(&*gpu); let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { + let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); + let raw_stats = match crate::sysfs_exports::register_g17( + raw_dev, + snapshot, + module_parameters::stats_export.value() != 0, + ) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); return Err(error); } }; + if let Err(error) = + crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) + { + dev_warn!( + pdev.as_ref(), + "idle_off_delay_ms not registered: {:?}\n", + error + ); + } Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, + raw_dev, + raw_stats, + stats, + }) + } +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl AsahiPlatformDriver { + pub(crate) fn probe_g16<'bound>( + pdev: &'bound platform::Device>, + cfg: &'static crate::g16::Config, + ) -> Result> { + // SAFETY: the qualified UAT output width constrains this device's DMA. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + let resources = regs::Resources::new(pdev)?; + let res = regs::G16Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; + let gpu = crate::g16::Gpu::new(pdev, &drm, cfg, res)?; + let data_gpu = gpu.clone() as Arc; + let data = try_pin_init!(AsahiDrmRegistrationData { + gpu: data_gpu, + resources, + }); + // SAFETY: registration is stored in bound platform data and never forgotten. + let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + let raw_dev = pdev.as_ref().as_raw(); + let raw_stats = crate::sysfs_exports::register(raw_dev, core::ptr::null(), false)?; + Ok(AsahiPlatformDriverData { + binding, + _device: reg.device().into(), + _reg: reg, + g17: None, + m3: None, + g16: Some(gpu), raw_dev, raw_stats, stats: None, @@ -431,9 +533,16 @@ impl AsahiPlatformDriver { } } +impl AsahiPlatformDriverData<'_> { + pub(crate) fn close_binding(&self) { + self.binding.close(); + } +} + impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { - // SAFETY: `self.raw_dev` was set in `probe` from the platform + self.binding.close(); + // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index f2f055931d29a..24e8db30e580e 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,6 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, + gpu, + memacct, mmu, queue, util::{ @@ -43,6 +45,7 @@ use kernel::sync::{ Mutex, // }; use kernel::time::NSEC_PER_SEC; +use kernel::types::ScopeGuard; use kernel::uaccess::{ UserPtr, UserSlice, // @@ -68,6 +71,7 @@ struct Vm { user_range: Range, kernel_range: Range, _dummy_mapping: mmu::KernelMapping, + gpu: Arc, } impl Drop for Vm { @@ -89,6 +93,7 @@ impl Drop for Vm { pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); } self.vm.bo_deferred_cleanup(); + self.gpu.vm_closed(&self.vm); return; } // When the user Vm is dropped, verify that no user mappings exists. @@ -122,10 +127,23 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, + pub(crate) cumulative: Option, + pub(crate) timeline: bool, } impl SyncItem { - fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + fn parse_one( + file: &DrmFile, + mut data: uapi::drm_asahi_sync, + out: bool, + bundled: bool, + ) -> Result { + if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !out || !bundled || data.timeline_value == 0 { + return Err(EINVAL); + } + data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; + } match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -146,6 +164,8 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -168,12 +188,10 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: if out { - Some(dma_fence::FenceChain::new()?) - } else { - None - }, + chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: true, }) } _ => { @@ -188,28 +206,52 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, + allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let mut anchor = None; - // SAFETY: We only read this once, so there are no TOCTOU issues. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); + // Read each record once so validation and use see the same retained bytes. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) })?; // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !allow_cumulative || i < in_count || anchor.is_some() { + return Err(EINVAL); + } + anchor = Some(i as usize); + } + vec.push( + Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, + GFP_KERNEL, + )?; + } + if let Some(anchor) = anchor { + let bundle = crate::g17::sync_output::SignalBundle::new( + &vec[in_count as usize..], + anchor - in_count as usize, + )?; + vec[anchor].cumulative = Some(bundle); + } else { + for sync in &mut vec[in_count as usize..] { + if sync.timeline { + sync.chain_fence = Some(dma_fence::FenceChain::new()?); + } + } } Ok(vec) @@ -218,7 +260,7 @@ impl SyncItem { #[derive(Clone)] pub(crate) enum Object { - TimestampBuffer(Arc), + TimestampBuffer(Arc), } /// State associated with a client. @@ -233,6 +275,10 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, + /// GPU memory this client created and holds. + mem: Arc, + /// Completed engine time, retained by queues after handle destruction. + usage: Arc, /// Largest size this file has asked GET_PARAMS for the global parameters with /// ([`gpu::Gpu::admit_vm`]). params_size: core::sync::atomic::AtomicU64, @@ -255,12 +301,35 @@ impl drm::file::DriverFile for File { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); let user_range = gpu.user_range(); + let mem = memacct::Client::new()?; + let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) + KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) + }) + } + + /// The objects this client created, which `memacct` attributes to it, not every object it + /// has a handle to: an imported or shared buffer is counted once, by its creator. + fn memory_stats(&self) -> Option { + Some(drm::file::MemoryStats { + private: self.mem.committed(), + resident: Some(self.mem.resident()), + ..Default::default() }) } + fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { + let names = [ + kernel::c_str!("vertex"), + kernel::c_str!("fragment"), + kernel::c_str!("compute"), + ]; + for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { + emit(name, ns); + } + } + fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -270,25 +339,41 @@ impl drm::file::DriverFile for File { unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} impl File { - fn new(id: u64, user_range: Range) -> impl PinInit { + fn new( + id: u64, + user_range: Range, + mem: Arc, + usage: Arc, + ) -> impl PinInit { + // SAFETY: The initializer supplies aligned, pinned storage for Self. + // XArray::new is infallible; each pinned array is initialized in place, + // then every remaining field is written before returning success. unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_vms)?; + pin_init::raw_try_init( + raw_vms, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; let raw_queues = addr_of_mut!((*slot).queues); - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ) - .__pinned_init(raw_queues)?; + pin_init::raw_try_init( + raw_queues, + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ), + )?; let raw_objects = addr_of_mut!((*slot).objects); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_objects)?; + pin_init::raw_try_init( + raw_objects, + xarray::XArray::>::new(xarray::AllocKind::Alloc1), + )?; (*slot).id = id; (*slot).user_range = user_range; + addr_of_mut!((*slot).mem).write(mem); + addr_of_mut!((*slot).usage).write(usage); addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); Ok(()) }) @@ -341,7 +426,7 @@ impl File { let limits = gpu.queue_limits().ok_or(EINVAL)?; let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; UserSlice::new( - UserPtr::from_addr(data.pointer as _), + UserPtr::from_addr(data.pointer as usize), core::mem::size_of_val(&words), ) .writer() @@ -360,8 +445,11 @@ impl File { return Err(ENODEV); } + let memory_limits = memacct::LEDGER.limits(); let mut params = uapi::drm_asahi_params_global { features: 0, + memory_limit: memory_limits.device, + client_memory_limit: memory_limits.client, vm_start: file.inner().user_range.start, vm_end: file.inner().user_range.end, @@ -381,13 +469,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + // Write the retained result to this user slice once. let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) + core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) })?; file.inner() @@ -402,15 +490,18 @@ impl File { if data.pad != 0 || data.size < size as u64 { return Err(EINVAL); } - let user = UserPtr::from_addr(data.pointer as _); + let user = UserPtr::from_addr(data.pointer as usize); let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; if flags != 0 || pad != 0 { return Err(EINVAL); } let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size) - .writer() - .write(&[vm_id, 0, status.reported_error() as u32, 0])?; + UserSlice::new(user, size).writer().write(&[ + vm_id, + 0, + status.reported_error() as u32, + 0, + ])?; Ok(0) } @@ -421,7 +512,7 @@ impl File { return Err(EINVAL); } let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size) + UserSlice::new(UserPtr::from_addr(data.pointer as usize), size) .reader() .read::<[u32; 4]>()?; if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { @@ -492,7 +583,22 @@ impl File { let gpu = ®_data.gpu; let file_id = file.inner().id; - let vm = gpu.new_vm(kernel_range.clone())?; + let vm = gpu.new_vm_for_file(kernel_range.clone(), file_id)?; + // G16 registers a retained backend owner before returning this VM. Any later + // allocation can fail before file::Vm exists to run its normal close path. + let failed_vm = ScopeGuard::new_with_data( + (vm.clone(), user_range.clone(), kernel_range.clone()), + |(vm, user_range, kernel_range)| { + if gpu.tracked_submission() && vm.status().is_some() && !vm.is_closed() { + vm.close_idle_reclaim(); + if let Err(error) = vm.unmap_user_ranges(user_range, kernel_range) { + pr_err!("VM create: closing unpublished VM failed: {:?}\n", error); + } + vm.bo_deferred_cleanup(); + gpu.vm_closed(&vm); + } + }, + ); let vm_xa = file.inner().vms(); let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -568,10 +674,12 @@ impl File { user_range, kernel_range, _dummy_mapping: dummy_mapping, + gpu: gpu.clone(), }, GFP_KERNEL, )?)?; + drop(failed_vm.dismiss()); data.vm_id = id; Ok(0) @@ -580,7 +688,7 @@ impl File { /// IOCTL: vm_destroy: Destroy a `Vm`. pub(crate) fn vm_destroy( _device: &AsahiDrmDevice, - reg_data: &driver::AsahiDrmRegistrationData<'_>, + _reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_vm_destroy, file: &DrmFile, ) -> Result { @@ -589,7 +697,6 @@ impl File { Err(ENOENT) } else { drop(vm); - reg_data.gpu.vm_closed(); Ok(0) } } @@ -632,11 +739,48 @@ impl File { None }; - let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; - + let size = gem::checked_object_size(data.size.try_into()?)?; + let commit = memacct::LEDGER + .admit(&file.inner().mem, size as u64) + .inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; + // Large G17 user objects are populated in full by the first VM_BIND. + // Do not multiply direct reclaim/compaction latency by their folio + // count. Userspace can evict its BO cache and retry after ENOMEM. + let pages_no_reclaim = + gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); + let gem = gem::new_object( + device, + size, + data.flags, + resv_obj, + reclaim_cpu, + pages_no_reclaim, + commit, + )?; if reg_data.gpu.prepare_new_objects() { - gem::prepare_new_object(&gem)?; + #[cfg(CONFIG_DRM_ASAHI_G15)] + if reg_data.gpu.as_any().is::() { + gem::prepare_new_object_g15(&gem)?; + } else { + let registered = device.registration_guard().ok_or(ENODEV)?; + let parent: &kernel::platform::Device = registered.as_ref(); + gem::prepare_new_object(&gem, parent.as_ref())?; + } + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + let registered = device.registration_guard().ok_or(ENODEV)?; + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + { + let parent: &kernel::platform::Device = registered.as_ref(); + gem::prepare_new_object(&gem, parent.as_ref())?; + } } + let handle = gem.create_handle(file)?; data.handle = handle; @@ -713,11 +857,17 @@ impl File { } if let Some(limit) = reg_data.gpu.bind_batch_limit() { - if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 - || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { + if data.num_binds == 0 + || data.num_binds > limit + || data.userptr == 0 + || data.stride % 8 != 0 + || data.stride as usize > kernel::page::PAGE_SIZE + { + cls_pr_debug!(Errors, "vm_bind: Batch outside the GPU's limits\n"); return Err(EINVAL); } } + if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -728,7 +878,7 @@ impl File { let size = usize::try_from(data.stride)? .checked_mul(usize::try_from(data.num_binds)?) .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as usize), size).reader(); reader.read_all(&mut vec, GFP_KERNEL)?; let mut reader = Reader::new(&vec); @@ -746,7 +896,7 @@ impl File { let stride = usize::try_from(data.stride)?; let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as _), size) + UserSlice::new(UserPtr::from_addr(data.userptr as usize), size) .reader() .read_all(&mut bytes, GFP_KERNEL)?; let (vm, user_range, kernel_range) = { @@ -1194,11 +1344,33 @@ impl File { return Err(EINVAL); } - // TODO: Allow with CAP_SYS_NICE - if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { + // REALTIME stays reserved. HIGH needs CAP_SYS_NICE or DRM master, as amdgpu and + // panthor require for raised queue priority; EACCES lets userspace fall back to MEDIUM. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); return Err(EINVAL); } + // HIGH has only been run on G17; G13 and G14 keep refusing it. + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + && !reg_data.gpu.as_any().is::() + { + cls_pr_debug!( + Errors, + "queue_create: HIGH priority is not supported on this GPU\n" + ); + return Err(EINVAL); + } + if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH + // SAFETY: capable() reads the current ioctl task's credentials. + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } + && !file.is_current_master() + { + cls_pr_debug!( + Errors, + "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" + ); + return Err(EACCES); + } let queues_xa = file.inner().queues(); let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; @@ -1219,6 +1391,7 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, + file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; @@ -1260,6 +1433,9 @@ impl File { } let gpu = ®_data.gpu; + if data.cmdbuf_size as usize > gpu.max_cmdbuf_size() { + return Err(E2BIG); + } if let Some(manager) = gpu.manager() { manager.update_globals(); } @@ -1292,7 +1468,7 @@ impl File { let mut vec = KVec::new(); let copy_commands = |bytes: &mut KVec| { UserSlice::new( - UserPtr::from_addr(data.cmdbuf as _), + UserPtr::from_addr(data.cmdbuf as usize), data.cmdbuf_size as usize, ) .reader() @@ -1304,8 +1480,13 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + let syncs = SyncItem::parse_array( + file, + data.syncs, + data.in_sync_count, + data.out_sync_count, + gpu.as_any().is::(), + )?; mod_dev_dbg!( device, @@ -1369,7 +1550,7 @@ impl File { // TODO: Do this on device-init for perf. let gpu = ®_data.gpu; - let frequency_hz = gpu.base_clock_hz() as u64; + let frequency_hz = u64::from(gpu.base_clock_hz()); let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); let num = (NSEC_PER_SEC as u64) / ts_gcd; diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index a740ad3bb945d..51cac02624d3d 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -44,6 +44,11 @@ struct F32U { } impl F32 { + /// Return the IEEE-754 binary32 representation. + pub(crate) const fn to_bits(self) -> u32 { + self.0 + } + /// Convert a raw 32-bit representation into an F32 pub(crate) const fn from_bits(u: u32) -> F32 { F32(u) @@ -140,12 +145,12 @@ macro_rules! from_ints { ($u:ty, $i:ty) => { impl From<$i> for F32 { fn from(v: $i) -> F32 { - F32U::from_i64(v as i64).pack() + F32U::from_i64(i64::from(v)).pack() } } impl From<$u> for F32 { fn from(v: $u) -> F32 { - F32U::from_u64(v as u64).pack() + F32U::from_u64(u64::from(v)).pack() } } }; diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 324bc08194e53..ae0512de0b156 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait -//! for the native display and PMP supplier before booting firmware. T6031 retains explicit -//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for +//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting +//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -45,6 +45,7 @@ mod adt; mod board; mod boot; +mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -143,22 +144,23 @@ const OF_TABLE: kernel::device_id::IdArray = kernel::device_id::IdArray::new([ ( of::DeviceId::new(c_str!("apple,agx-t6030")), - &hw::t6030::CONFIG, + &&hw::t6030::CONFIG, ), ( of::DeviceId::new(c_str!("apple,agx-t6031")), - &hw::t6031::CONFIG, + &&hw::t6031::CONFIG, ), ]); -// T6030 probe admission requires the native display and PMP supplier before -// firmware starts. T6031 retains explicit loading until it has an equivalent -// display-order admission gate. +// Probe admission requires the native display before firmware starts (T6030 +// also its PMP supplier), so both may be loaded at device creation. kernel::of_device_table!( AUTOLOAD_OF_TABLE, - MODULE_OF_TABLE_G15, - (), - [(of::DeviceId::new(c_str!("apple,agx-t6030")), ())] + @none, + [ + of::DeviceId::new(c_str!("apple,agx-t6030")), + of::DeviceId::new(c_str!("apple,agx-t6031")), + ] ); /// The G15 platform adapter: the shared DRM registration probed through @@ -177,6 +179,12 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + if let Some(driver) = &this._driver { + driver.close_binding(); + } + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true @@ -211,7 +219,6 @@ impl platform::Driver for Driver<'_> { }, }) } - } /// Firmware starts only after every fallible module/driver registration has succeeded. @@ -222,6 +229,8 @@ static MODULE_READY: AtomicBool = AtomicBool::new(false); /// No fallible module initialization may follow this call. pub(crate) fn activate() { MODULE_READY.store(true, Ordering::Release); + // SAFETY: The helper only reads the device tree. + unsafe { boot::glue::asahi_g15_report_loader_gpu() }; // SAFETY: Both pinned platform registrations are fully initialized and remain owned // by DriverModule. The C helper borrows their live driver pointer only for this call. unsafe { boot::glue::asahi_g15_reprobe() }; @@ -372,7 +381,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. + /// Live VMs and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -437,7 +446,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G15: no aurora-g15 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { @@ -450,17 +459,18 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain(); + gpu.retain()?; Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once; `boot::start` already took it before touching the + /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) { - boot::glue::retain_module(); + pub(crate) fn retain(&self) -> Result { + boot::glue::retain_module() } } @@ -535,7 +545,21 @@ impl Firmware { vms, }, ); - if let Err(e) = firmware.init_graphs() { + let result = firmware.init_graphs().and_then(|_| { + if cfg.platform != hw::Platform::AdtWithPmp { + return Ok(()); + } + // A report before firmware startup is ACKed without enabling bandwidth scaling. + // Reassert it after setup, including the backend's final device control. + // SAFETY: This session retains its acknowledged T6030 AGX power vote. + let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; + if ret == 0 { + Ok(()) + } else { + Err(Error::from_errno(ret)) + } + }); + if let Err(e) = result { let Firmware { session, init, @@ -630,7 +654,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_closed(&self) { + fn vm_closed(&self, _vm: &mmu::Vm) { vm::closed(self) } @@ -645,9 +669,10 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base)?, + queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, GFP_KERNEL, )?) } @@ -656,7 +681,7 @@ impl gpu::Gpu for Gpu { &self, bo: gem::ObjectRef, range: Range, - ) -> Result { - timestamp::map(self, bo, range) + ) -> Result { + crate::timestamp::Buffer::firmware(timestamp::map(self, bo, range)?) } } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 3855bd0beb9cb..0f91f7c9f0610 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,8 +25,10 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` -//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) +//! and return them after successful retirement. Failed published jobs retain their slots until +//! their last reference drops (until reboot for jobs retained after a fail-stop). +//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; @@ -96,7 +98,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(pa, count, f) } + unsafe { pgtable::with_root_entries(None, pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index a06b5da5ed6b1..3da16c10696e3 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,7 +11,6 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. -use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -36,6 +35,7 @@ use kernel::{ }, macros::versions, new_mutex, + new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -45,6 +45,7 @@ use kernel::{ }, Arc, Mutex, + SpinLock, UniqueArc, // }, time::{ @@ -85,6 +86,7 @@ use crate::{ queue, regs, stats, + stats_raw, workqueue, // }; @@ -259,18 +261,30 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, + /// Last raw firmware statistics of this device, exported read-only + /// through `stats_raw`. + raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, - #[allow(clippy::vec_box)] #[pin] - garbage_contexts: Mutex>>>, + garbage_contexts: Mutex>, } /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { + /// Admission bound checked before copying the userspace command stream. + fn max_cmdbuf_size(&self) -> usize { + usize::MAX + } + /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; + /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. + fn raw_stats_snapshot(&self) -> Option>> { + None + } + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -285,6 +299,10 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). + fn supports_soft_faults(&self) -> bool { + crate::module_parameters::fault_control.value() == 0xb + } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false @@ -297,14 +315,14 @@ pub(crate) trait Gpu: Send + Sync { fn feature_flags(&self) -> u64 { let mut flags = 0; if self.supports_vm_status() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR); } if self.queue_limits().is_some() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES); } - if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + if self.soft_faults() && self.supports_soft_faults() { + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); } flags } @@ -312,35 +330,38 @@ pub(crate) trait Gpu: Send + Sync { fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; - /// Backend-specific user address aperture; other GPUs keep their geometry's range. + /// The VA range that user VMs may use (`vm_start..vm_end` of GET_PARAMS). fn user_range(&self) -> Range { self.uat_geometry().user_usable_range() } - /// Use tracked command publication and sticky VM status on this backend. + /// Whether this GPU uses the tracked submission model of `file.rs` (besides G17, which + /// `file.rs` recognizes by type): VM_BIND batches are validated as a whole against + /// driver-reserved ranges before any change, and SUBMIT copies the command buffer before + /// resolving syncs because the queue installs a fence in every output sync even on failure. fn tracked_submission(&self) -> bool { false } - /// Whether faults may be reported as recoverable to userspace. + /// Whether SOFT_FAULTS may be advertised when `fault_control` selects soft faults. fn soft_faults(&self) -> bool { true } - /// Populate and clean fresh object backing before any WC CPU mapping. + /// Whether new user objects must be populated, cleaned to the point of coherency and pinned + /// at creation, before any CPU view exists (GPUs whose device is not DMA-coherent and whose + /// default CPU mappings are write-combined). fn prepare_new_objects(&self) -> bool { false } - /// Optional backend-specific upper bound for a VM_BIND batch. + /// The largest VM_BIND batch, if the GPU's UAPI contract bounds it; with a bound, a batch also + /// needs at least one operation, a non-null pointer and a stride that is a multiple of 8 and at + /// most a page (EINVAL otherwise). `None` keeps the shared rules. fn bind_batch_limit(&self) -> Option { None } - /// A userspace VM handle was closed; release backend-only registry references. - fn vm_closed(&self) {} - /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS - /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 - /// T8122 runtime admits any client. + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; + /// Admit a client's parameter ABI before creating its VM. fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { Ok(()) } @@ -350,6 +371,13 @@ pub(crate) trait Gpu: Send + Sync { } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; + /// Create a VM charged to a DRM file when the backend bounds retained bindings. + fn new_vm_for_file(&self, kernel_range: Range, _file_id: u64) -> Result { + self.new_vm(kernel_range) + } + /// Release the backend's handle owner after closing user mapping admission. + /// Accepted jobs and firmware resource leases retain their own references. + fn vm_closed(&self, _vm: &mmu::Vm) {} /// Create a new user command queue. fn new_queue( &self, @@ -358,13 +386,14 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result>; - /// Map a BO as a timestamp buffer + /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( &self, bo: gem::ObjectRef, range: Range, - ) -> Result; + ) -> Result; } /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. @@ -406,8 +435,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_cfg(&self) -> &'static hw::HwConfig; /// Get the dynamic GPU configuration for this SoC. fn get_dyncfg(&self) -> &hw::DynConfig; - /// Register an unused context as garbage - fn free_context(&self, data: KBox>); + /// Register an unused context, with the objects it retains, for invalidation. + fn free_context(&self, ctx: workqueue::RetiredContext); } /// Private generic trait for functions that don't need to escape this module. @@ -752,6 +781,7 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, + mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) @@ -816,6 +846,12 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); + // Per-device raw statistics snapshot, shared with the debugfs export. + let raw_stats_snap = Arc::pin_init( + new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), + GFP_KERNEL, + )?; + let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -831,6 +867,7 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, + raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -858,6 +895,7 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), + raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1089,7 +1127,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(map.writable as u64), + readwrite: U64(u64::from(map.writable)), }; }); @@ -1139,20 +1177,14 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let Some(guard) = self.dev.registration_guard() else { - return None; - }; + let guard = self.dev.registration_guard()?; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if info.is_some() { - dev_err!( - self.dev.as_ref(), - " Fault info: {:#x?}\n", - info.as_ref().unwrap() - ); + if let Some(fault) = &info { + dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); } info }) @@ -1319,6 +1351,10 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } + fn raw_stats_snapshot(&self) -> Option>> { + Some(self.raw_stats.clone()) + } + fn as_any(&self) -> &dyn Any { self } @@ -1401,6 +1437,7 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1418,6 +1455,7 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, + usage, )?, GFP_KERNEL, )?) @@ -1427,15 +1465,15 @@ impl Gpu for GpuManager::ver { &self, mut bo: gem::ObjectRef, range: Range, - ) -> Result { - bo.map_range_into_range( + ) -> Result { + crate::timestamp::Buffer::firmware(bo.map_range_into_range( self.uat.kernel_vm(), range, IOVA_KERN_TIMESTAMP_RANGE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false, - ) + )?) } } @@ -1460,7 +1498,7 @@ impl GpuManager for GpuManager::ver { core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); for ctx in garbage_ctx { - if self.invalidate_context(&ctx).is_err() { + if self.invalidate_context(&ctx.data).is_err() { dev_err!( self.dev.as_ref(), "GpuContext: Failed to invalidate GPU context!\n" @@ -1469,6 +1507,9 @@ impl GpuManager for GpuManager::ver { panic!("GPU firmware timed out"); } } + // The firmware no longer knows the context's queues, so their structures may go now, + // ahead of the context data they point to. + core::mem::drop(ctx.retained); } let mut guard = self.alloc.lock(); @@ -1737,7 +1778,7 @@ impl GpuManager for GpuManager::ver { &self.dyncfg } - fn free_context(&self, ctx: KBox>) { + fn free_context(&self, ctx: workqueue::RetiredContext) { let mut garbage = self.garbage_contexts.lock(); if garbage.push(ctx, GFP_KERNEL).is_err() { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index dc828a460202d..437685bfad1de 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -15,7 +15,7 @@ //! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; -use core::mem::size_of; +use core::mem::{size_of, ManuallyDrop}; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{ @@ -29,7 +29,6 @@ use core::sync::atomic::{ use kernel::{ addr::{ - DmaAddr, PhysicalAddr, // }, c_str, @@ -109,20 +108,19 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; -use pin_init; - const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; -mod shared; mod lifetime; +mod ppl; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod m3; mod residency; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; +mod shared; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; @@ -156,23 +154,81 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// Two coprocessor instances that never touch the handoff region (G17). + /// One or two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, the secondary one the root at - /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel - /// context. + /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at + /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. + /// TTBAT context 1 starts as an alias of the kernel context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: u64, + secondary_root_offset: Option, }, } +/// UAT policy of a SoC where it differs from the A18 Pro's. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UatQuirks { + /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and + /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of + /// `struct page`. + /// + /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the + /// regions lose `no-map` and this path goes away. + pub(crate) reserved_tables: bool, + // The following settings keep a SoC on the only policy it was tested with. Each one is + // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. + /// Write back every page-table and context-table change: the GPU table walker is not + /// assumed to snoop the CPU caches. + pub(crate) clean_tables: bool, + /// Invalidate ranges one page at a time, never with range TLBI operations. + pub(crate) page_tlbi: bool, + /// Take the handoff lock around context-table edits. Where the firmware leaves its half of + /// the handoff page alone, its lock bytes are not initialized and are never taken. + pub(crate) handoff_lock: bool, + /// Handoff "current slot" written at initialization. + pub(crate) handoff_slot: u32, + /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in + /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. + pub(crate) native_gpu_permissions: bool, +} + +impl UatQuirks { + /// The A18 Pro and earlier policy. + pub(crate) const DEFAULT: UatQuirks = UatQuirks { + reserved_tables: false, + clean_tables: false, + page_tlbi: false, + handoff_lock: true, + handoff_slot: 0, + native_gpu_permissions: false, + }; +} + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; +/// M4's nonzero wire incarnation of a host-owned root installation. +const fn binding_wire_generation(incarnation: u64) -> u8 { + if incarnation == 0 { + 0 + } else { + (1 + (incarnation - 1) % 255) as u8 + } +} + +/// Upper word of an application TTBAT slot. M4 exposes firmware upper mappings only in +/// context 0; the existing G15 handoff shares them with user contexts. +const fn application_ttb1(root: u64, asid: usize, map_kernel_to_user: bool) -> u64 { + if map_kernel_to_user { + root | TTBR_VALID | (asid as u64) << TTBR_ASID_SHIFT + } else { + 0 + } +} + /// UAT address space geometry of a given SoC. #[derive(Debug, Copy, Clone)] pub(crate) struct UatGeometry { @@ -312,6 +368,7 @@ struct VmInner { va_range: Range, page_table: UatPageTable, mapping_epoch: Option>, + bind_sequence: Arc, epoch: Option, mm: mm::Allocator<(), KernelMappingInner>, uat_inner: Arc, @@ -357,19 +414,54 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +pub(crate) struct VmReadGuard(VmExecGuard); + +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +impl VmReadGuard { + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { + Vm::covers_range_locked(&mut self.0, address, size, read, write) + } + + pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { + Vm::read_user_locked(&mut self.0, va, buf) + } +} + /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, binding: Option>, bind_token: Option, + /// Nonzero M4 wire incarnation of the root currently owning this slot. + generation: u8, ttb: u64, } /// Data associated with a VM <=> BO pairing #[pin_data] struct VmBoData { + /// Highest sequence assigned to a GPUVA linked to this VM/BO pair. + /// + /// Map steps update this monotonically under the VM reservation before + /// linking the GPUVA. Remap pieces keep their original sequence. + newest_bind: AtomicU64, #[pin] - segments: Mutex>>, + segments: Mutex>, +} + +/// Copied DMA addresses and the lease covering every alias of this VM/BO association. +/// The lease is dropped only after all aliases have been unmapped and invalidated. +struct SgSegments { + segments: KVVec<(DmaAddress, ResourceSize)>, + _lease: kernel::drm::gem::shmem::SGLease, +} + +impl SgSegments { + fn iter(&self) -> core::slice::Iter<'_, (DmaAddress, ResourceSize)> { + self.segments.iter() + } } /// Preallocated resources needed to execute a VM operation. @@ -410,6 +502,9 @@ struct StepContext<'ctx> { /// Preallocated resources that can be used when executing the request. resources: &'ctx mut VmOpResources, + + /// Sequence assigned by the successful map for its related metadata. + bind_sequence: u64, } impl<'ctx> StepContext<'ctx> { @@ -422,6 +517,7 @@ impl<'ctx> StepContext<'ctx> { dev, prot, resources, + bind_sequence: 0, }) } @@ -452,6 +548,7 @@ impl<'ctx> StepContext<'ctx> { #[derive(Default, Clone)] struct VmGpuVa { prot: Option, + bind_sequence: u64, } impl gpuvm::DriverGpuVm for VmInner { @@ -470,6 +567,11 @@ impl gpuvm::DriverGpuVm for VmInner { op: OpMap<'op, Self>, ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { + let bind_sequence = self.bind_sequence.fetch_update( + Ordering::AcqRel, + Ordering::Acquire, + |value| value.checked_add(1), + ).map_err(|_| EOVERFLOW)? + 1; let _mutation = self.mapping_mutation(); let (map_addr, map_length) = (op.addr(), op.length() as usize); let mut iova = op.addr(); @@ -478,22 +580,16 @@ impl gpuvm::DriverGpuVm for VmInner { let one_page = op.flags().contains(gpuvm::GpuVaFlag::REPEAT); - let Some(dev) = ctx.dev else { - dev_err!(self.dev.as_ref(), "StepContext missed Bound device!"); + if ctx.dev.is_none() { return Err(ENODEV); - }; - - let sgt = op.obj().sg_table(dev).inspect_err(|e| { - dev_err!( - self.dev.as_ref(), - "sm_step_map() Failed to get SG table: {:?}", - e - ); - })?; - for sgt_entry in sgt.iter() { - // TODO: proper DMA address/length handling - let mut addr = sgt_entry.dma_address() as usize; - let mut len: usize = sgt_entry.dma_len() as usize; + } + // Bind preparation retained the table's backing before taking the VM reservation. + // Never acquire the object reservation from inside the GPUVM map callback. + let segments = op.vm_bo().data().segments.lock(); + let sgt = segments.as_ref().ok_or(EFAULT)?; + for &(dma_address, dma_len) in sgt.iter() { + let mut addr = dma_address as usize; + let mut len = dma_len as usize; if left == 0 { break; @@ -540,6 +636,7 @@ impl gpuvm::DriverGpuVm for VmInner { iova += len as u64; } + drop(segments); let gpuva = match ctx.preallocated_gpuva() { Ok(gpuva) => gpuva, Err(error) => { @@ -547,12 +644,18 @@ impl gpuvm::DriverGpuVm for VmInner { return Err(error); } }; + op.vm_bo() + .data() + .newest_bind + .fetch_max(bind_sequence, Ordering::Relaxed); let op = op.insert( gpuva, VmGpuVa { prot: Some(ctx.prot), + bind_sequence, }, ); + ctx.bind_sequence = bind_sequence; self.tlbi_contexts_naming_root(map_addr, map_length); Ok(op) @@ -578,6 +681,7 @@ impl gpuvm::DriverGpuVm for VmInner { ctx: &mut Self::SmContext<'ctx>, ) -> Result, Error> { let orig_prot = op.va_to_unmap().data().prot.ok_or(EINVAL)?; + let orig_bind_sequence = op.va_to_unmap().data().bind_sequence; let orig_invalidated = op.va_to_unmap().flags().contains(GpuVaFlag::INVALIDATED); // Only unmap the hole between prev/next, if they exist @@ -618,7 +722,8 @@ impl gpuvm::DriverGpuVm for VmInner { let _mutation = self.mapping_mutation(); if orig_invalidated { - self.page_table.discard_partial_map(unmap_start..unmap_end)?; + self.page_table + .discard_partial_map(unmap_start..unmap_end)?; } else { self.page_table.unmap_pages(unmap_start..unmap_end)?; } @@ -642,9 +747,11 @@ impl gpuvm::DriverGpuVm for VmInner { [prev_gpuva, next_gpuva], VmGpuVa { prot: Some(orig_prot), + bind_sequence: orig_bind_sequence, }, VmGpuVa { prot: Some(orig_prot), + bind_sequence: orig_bind_sequence, }, ); @@ -659,7 +766,12 @@ impl gpuvm::DriverGpuVm for VmInner { impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.length()); + mod_dev_dbg!( + self.dev, + "MMU: unmap: {:#x}:{:#x}\n", + va.addr(), + va.length() + ); let _mutation = self.mapping_mutation(); if va.flags().contains(GpuVaFlag::INVALIDATED) { @@ -694,14 +806,7 @@ impl VmInner { // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - mem::sync(); - self.tlbi_range(asid as u8, addr, size as usize); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size as usize); + self.finish_failed_map(addr, size as usize); if let Err(error) = result { dev_err!( self.dev.as_ref(), @@ -711,6 +816,19 @@ impl VmInner { } } + /// Complete discarded stores and invalidate both the legacy bound slot and + /// every M4/G17 context naming this root. Call only after releasing PPL. + fn finish_failed_map(&self, addr: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + mem::sync(); + self.tlbi_range(asid as u8, addr, size); + mem::sync(); + } + } + self.tlbi_contexts_naming_root(addr, size); + } + fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { @@ -765,15 +883,19 @@ impl VmInner { /// /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Other firmware only uses the bind slot, which the callers - /// already invalidate. + /// change must reach all of them. Adopted M4 roots also remain in the TTBAT after a bind + /// lease is dropped, until the slot is reassigned or the root is destroyed. Include those + /// contexts before allowing an unmapped object's backing to be released. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { + if self.uat_inner.firmware == UatFirmware::Handoff && !self.uat_inner.m4_adopted { mem::sync(); return; } - let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + let contexts = self + .uat_inner + .lock() + .contexts_naming_root(self.ttb(), self.uat_inner.m4_adopted); self.tlbi_context_mask(iova, size, contexts); } @@ -801,10 +923,35 @@ impl VmInner { .checked_add(node.mapped_size as u64) .ok_or(EOVERFLOW)?; let _mutation = self.mapping_mutation(); + // Allocate detached RAM tables before excluding the firmware. The shadow L2 is not + // the reserved shared-L2; only final leaf stores below already linked L3s are live. self.page_table.prepare_map(node.start()..end)?; + let uat_inner = self.uat_inner.clone(); + let handoff = uat_inner.lock_m4_upper(self.is_kernel)?; let result = self.map_node_prepared(node, prot); + let discard = if result.is_err() { + // Rollback still changes live leaves and must remain under the PPL lease. + self.page_table.discard_partial_map(node.start()..end) + } else { + Ok(()) + }; + // The M4 handoff guard owns UatShared's mutex. Invalidation follows its release. + drop(handoff); if result.is_err() { - self.rollback_failed_map(node.start(), node.mapped_size as u64); + self.finish_failed_map(node.start(), node.mapped_size); + } else { + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + } + if let Err(error) = discard { + if self.uat_inner.m4_adopted { + // The caller retains its node and backing when this latch is set. + self.uat_inner.failed.store(true, Ordering::Release); + } + dev_err!( + self.dev.as_ref(), + "Could not discard failed kernel mapping: {:?}\n", + error + ); } result } @@ -812,7 +959,14 @@ impl VmInner { /// Install only into the empty range reserved by `map_node`. fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let segments = node.bo.as_ref().ok_or(EINVAL)?.get()?.data().segments.lock(); + let segments = node + .bo + .as_ref() + .ok_or(EINVAL)? + .get()? + .data() + .segments + .lock(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -870,8 +1024,7 @@ impl VmInner { } } - // A context naming this root may be live and hold a negative translation for the range. - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + // The caller invalidates negative translations after releasing its PPL lease. Ok(()) } } @@ -879,9 +1032,11 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { + device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, + bind_sequence: Arc, lifetime: Option>, residency: Option>, context_bindings: Option>, @@ -948,7 +1103,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -956,8 +1111,9 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); } - inner.handoff().unlock(); + inner.handoff_unlock(); owned || (low == 0 && high == 0) }; if release { @@ -998,6 +1154,12 @@ impl VmBind { pub(crate) fn slot(&self) -> u32 { self.1 } + + /// Stable wire incarnation for this active root lease. M4 reserves zero as unbound. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn generation(&self) -> u8 { + self.0.binding.lock().generation + } } impl Drop for VmBind { @@ -1038,10 +1200,7 @@ struct MappingBo { owner: ARef>, } impl MappingBo { - fn new( - bo: ARef>, - owner: ARef>, - ) -> Self { + fn new(bo: ARef>, owner: ARef>) -> Self { Self { bo: Some(bo), owner, @@ -1084,29 +1243,90 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// Prepared CPU timestamp access; destruction happens after completion signalling. +/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's +/// reservation lock, so it must not be dropped where fence producers run. +pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); + +impl TimestampVmap { + pub(crate) fn new(gem: &gem::Object) -> Result { + Ok(Self(gem.owned_vmap::<0>()?)) + } + + fn size(&self) -> usize { + use kernel::drm::gem::BaseObject; + self.0.owner().size() + } + + /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete + /// atomic store; byte stores support unaligned destinations without creating mutable + /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused + /// when the destination is prepared, and ignored here. + pub(crate) fn write(&self, offset: usize, value: u64) { + if offset + .checked_add(size_of::()) + .is_none_or(|end| end > self.size()) + { + return; + } + // Completion runs in task context. Bindings of the same BO share this lock, + // so aligned u64 and overlapping unaligned byte stores never race in Rust. + let _guard = self.0.owner().timestamp_lock.lock(); + let base = self.0.addr.cast::(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. + let pointer = unsafe { base.add(offset) }; + if pointer as usize % core::mem::align_of::() == 0 { + // SAFETY: All eight bytes were checked above; the vmap pins the backing. + // A naturally aligned query result becomes available in one complete store. + let destination = unsafe { &*pointer.cast::() }; + destination.store(value.to_le(), Ordering::Relaxed); + } else { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: The complete eight-byte extent was checked above. The owned + // vmap pins its backing; byte atomics permit unaligned and overlapping + // timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(offset + index).cast::() }; + destination.store(byte, Ordering::Relaxed); + } + } + // A CPU-retired query can be consumed by a later GPU command. WRITEBACK + // objects use a cacheable vmap, while GPU mappings need not snoop that cache. + // Publish the bytes before signalling the completion fence or queue frontier. + if self.0.owner().is_writeback() { + crate::pgtable::clean_poc(pointer, size_of::()); + } + crate::pgtable::poc_barrier(); + } +} + +/// Prepared CPU timestamp access with its own vmap; destruction happens after completion +/// signalling. +#[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) struct TimestampMapping { - mapping: kernel::drm::gem::shmem::VMapOwned, + mapping: TimestampVmap, offset: usize, } +#[cfg(CONFIG_DRM_ASAHI_G16)] impl TimestampMapping { - pub(crate) fn write(&self, value: u64) { - let base = self.mapping.addr.cast::(); - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { - &*base.add(self.offset + index).cast::() - }; - destination.store(byte, Ordering::Relaxed); + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); } + Ok(Self { + mapping: TimestampVmap::new(gem)?, + offset, + }) + } + + pub(crate) fn write(&self, value: u64) { + self.mapping.write(self.offset, value); } } /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); +pub(crate) struct KernelMapping(ManuallyDrop>); impl KernelMapping { /// Returns the IOVA base of this mapping @@ -1130,14 +1350,22 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - lock_vm(&self.0.owner, None, false)?.page_table.translate_iova(address) + self.0.owner.lock_inner().page_table.translate_iova(address) } - /// Writes a userspace timestamp through the retained GEM mapping. Byte - /// stores support unaligned destinations and concurrent completion writes - /// without creating mutable aliases. The complete timestamp is not atomic. + /// Writes a userspace timestamp through the retained GEM mapping. Aligned results + /// use a complete atomic store; byte stores support unaligned destinations without + /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + TimestampMapping::new(gem, self.timestamp_offset(offset)?) + } + + /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete + /// extent lies inside both the mapping and the object. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1148,10 +1376,12 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - Ok(TimestampMapping { - mapping: gem.owned_vmap::<0>()?, - offset: start, - }) + Ok(start) + } + + /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. + pub(crate) fn timestamp_vmap(&self) -> Result { + TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1263,13 +1493,13 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff().lock(); + uat_inner.handoff_lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff().unlock(); + uat_inner.handoff_unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1359,6 +1589,19 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { + // An ASC-cached M4 mapping has no qualified cache-flush teardown. Even an unexpected + // failed construction must retain its complete node/backing instead of invoking the + // unrelated G13/G14 FwCtl protocol. + if self.0.uat_inner.m4_adopted && self.0.prot.is_cached_noncoherent() { + self.0.uat_inner.failed.store(true, Ordering::Release); + pr_err!( + "MMU: retaining cached M4 mapping {:#x}+{:#x} until reboot\n", + self.iova(), + self.size() + ); + return; + } + // This is the main unmap function for UAT mappings. // The sequence of operations here is finicky, due to the interaction // between cached GFX ASC mappings and the page tables. These mappings @@ -1401,8 +1644,27 @@ impl Drop for KernelMapping { self.size() ); + let upper_handoff = match self.0.uat_inner.lock_m4_upper(owner.is_kernel) { + Ok(guard) => guard, + Err(error) => { + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after PPL failure: {:?}\n", + error + ); + return; + } + }; let _mutation = owner.mapping_mutation(); if owner.page_table.unmap_pages(self.iova_range()).is_err() { + if self.0.uat_inner.m4_adopted { + self.0.uat_inner.failed.store(true, Ordering::Release); + dev_err!( + owner.dev.as_ref(), + "MMU: retaining M4 mapping after partial unmap failure\n" + ); + return; + } dev_err!( owner.dev.as_ref(), "MMU: unmap {:#x}:{:#x} failed\n", @@ -1411,6 +1673,8 @@ impl Drop for KernelMapping { ); } + drop(upper_handoff); + if let Some(asid) = owner.slot() { mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); @@ -1427,6 +1691,11 @@ impl Drop for KernelMapping { // The range returns to the VA allocator once this mapping is gone, so no context that // names this root may keep a stale translation for it. owner.tlbi_contexts_naming_root(self.iova(), self.size()); + drop(_mutation); + drop(owner); + // SAFETY: Every normal path reaches this exactly once, after all GPUVM locks have + // been released. Uncertain M4 paths return earlier, retaining the node and its pins. + unsafe { ManuallyDrop::drop(&mut self.0) }; } } @@ -1435,32 +1704,73 @@ struct UatShared { kernel_ttb1: u64, execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], + /// Firmware-observed incarnations of application roots installed by the VM slot allocator. + binding_generations: [u64; UAT_NUM_CTX], + binding_published: [bool; UAT_NUM_CTX], + /// Expected lower words of user roots owned by the VM slot allocator. Replaced only by + /// successful host publications, never learned by accepting observed hardware words. + owned_user_roots: [u64; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, + quirks: UatQuirks, } impl UatShared { + /// Takes the handoff lock, where this SoC uses it. + fn handoff_lock(&self) { + if self.quirks.handoff_lock { + self.handoff().lock(); + } + } + + /// Releases the handoff lock taken by [`Self::handoff_lock`]. + fn handoff_unlock(&self) { + if self.quirks.handoff_lock { + self.handoff().unlock(); + } + } + + /// Writes back context-table slot `idx` after it changed, where this SoC needs it. + fn publish_slot(&self, idx: usize) { + if self.quirks.clean_tables { + let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); + pgtable::clean_poc(slot, size_of::()); + pgtable::poc_barrier(); + } + } + /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() + unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + unsafe { + self.ttbs_rgn + .map + .ptr() + .cast::<[SlotTTBS; UAT_NUM_CTX]>() + .as_ref() + } + .unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64) -> u64 { + fn contexts_naming_root(&self, ttb: u64, adopted: bool) -> u64 { self.ttbs() .iter() .enumerate() - .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) + .filter(|(ctx, slot)| { + let expected = tagged_root(ttb, *ctx); + let _ = adopted; // Context 0 is naturally untagged; user contexts carry their ASID. + slot.ttb0.load(Ordering::Acquire) == expected + }) .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) } } @@ -1477,20 +1787,138 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, - #[cfg(CONFIG_DRM_ASAHI_G15)] + /// The stopped J713 handoff: bounded full-barrier PPL and upper words zero in users. + m4_adopted: bool, + /// An uncertain mapping/root is retained; future M4 allocation and publication fail. + failed: AtomicBool, + /// Clean page tables to the point of coherency for a walker that does not snoop the CPU + /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. clean_poc: bool, + /// This SoC's UAT policy. + quirks: UatQuirks, + /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). + reserved: Option>, #[pin] shared: Mutex, #[pin] handoff_flush: [Mutex; UAT_NUM_CTX + 1], } +/// A transport owns the sole host-side PPL mutex before touching AP interest. +struct M4Transport<'a> { + shared: Guard<'a, UatShared, MutexBackend>, + start: Instant, + failed: &'a AtomicBool, +} + +impl ppl::Memory for M4Transport<'_> { + fn firmware_interested(&self) -> bool { + self.shared.handoff().lock_fw.load(Ordering::Relaxed) != 0 + } + fn firmware_turn(&self) -> bool { + self.shared.handoff().turn.load(Ordering::Relaxed) != 0 + } + fn ap_interest(&self, interested: bool) { + self.shared + .handoff() + .lock_ap + .store(u8::from(interested), Ordering::Relaxed); + } + fn give_firmware_turn(&self) { + self.shared.handoff().turn.store(1, Ordering::Relaxed); + } + fn barrier(&self) { + mem::sync(); + } + fn expired(&self) -> bool { + self.start.elapsed() > Delta::from_millis(100) + } + fn relax(&self) { + core::hint::spin_loop(); + } + fn fail(&self) { + self.failed.store(true, Ordering::Release); + } +} + +/// Every AP firmware transaction owns the same mutex, including G16 Window operations. +/// M4 drops its protocol guard before its transport releases the host mutex. The legacy +/// handoff path keeps the preexisting protocol and lock ordering for other generations. +#[must_use = "retain the guard until protected table mutations have completed"] +enum HandoffGuard<'a> { + M4(ppl::Guard>), + Legacy(Guard<'a, UatShared, MutexBackend>), +} + +impl HandoffGuard<'_> { + fn shared_mut(&mut self) -> &mut UatShared { + match self { + Self::M4(guard) => &mut guard.memory_mut().shared, + Self::Legacy(shared) => shared, + } + } +} + +impl Drop for HandoffGuard<'_> { + fn drop(&mut self) { + if let Self::Legacy(shared) = self { + shared.handoff_unlock(); + } + } +} + +/// Opaque host-and-firmware lease for G16 window transactions. No caller may +/// independently manipulate the AP-interest or turn words. +#[cfg(CONFIG_DRM_ASAHI_G16)] +#[must_use = "retain the guard throughout the reserved-table window transaction"] +pub(crate) struct M4HandoffGuard<'a> { + _transaction: HandoffGuard<'a>, +} + impl UatInner { /// Take the lock on the shared data and return the guard. fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { self.shared.lock() } + fn check_m4(&self) -> Result { + if self.m4_adopted && self.failed.load(Ordering::Acquire) { + Err(EIO) + } else { + Ok(()) + } + } + + /// Serialize AP callers before taking the firmware exclusion. Timeouts withdraw + /// AP interest while still holding the mutex and latch M4 failure before releasing it. + fn lock_handoff(&self) -> Result> { + self.check_m4()?; + let shared = self.lock(); + self.check_m4()?; + if !self.m4_adopted { + shared.handoff_lock(); + return Ok(HandoffGuard::Legacy(shared)); + } + let transport = M4Transport { + shared, + start: Instant::::now(), + failed: &self.failed, + }; + match ppl::Guard::acquire(transport) { + Ok(guard) => Ok(HandoffGuard::M4(guard)), + Err(()) => Err(ETIMEDOUT), + } + } + + /// No leaf is changed on a failed acquisition. Uncertain mappings retain their backing. + fn lock_m4_upper(&self, upper: bool) -> Result>> { + if upper && self.m4_adopted { + self.lock_handoff().map(Some) + } else { + Ok(None) + } + } + /// Take a lock on a handoff flush slot and return the guard. fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { self.handoff_flush[slot as usize].lock() @@ -1518,7 +1946,9 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 {} + while self.turn.load(Ordering::Relaxed) != 0 { + core::hint::spin_loop() + } self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -1543,9 +1973,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(0, Ordering::Relaxed); + self.cur_slot.store(slot, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -1625,19 +2055,39 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments( - dev: &device::Device, - gem: &gem::Object, -) -> Result> { +fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; - // self.segments. let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; for sgt_entry in sgt.iter() { segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(segments) + Ok(SgSegments { + segments, + _lease: sgt.into_lease(), + }) +} + +/// Reuse cached segments without allocating another copy for each bind. The bind lease was +/// acquired before BO locking: its getter synchronized with any earlier reclaim under resv, +/// and its extra count excludes new eviction until preparation/commit finishes. +fn prepare_sgt_segments( + dev: &device::Device, + gem: &gem::Object, + bo: &gpuvm::GpuVmBo, + _bind_lease: &kernel::drm::gem::shmem::SGLease, +) -> Result { + let needs_segments = bo.data().segments.lock().is_none(); + if needs_segments { + // Never enter the object reservation while holding the BO mutex. + let prefetched = prefetch_sgt_segments(dev, gem)?; + let mut segments = bo.data().segments.lock(); + if segments.is_none() { + segments.replace(prefetched); + } + } + Ok(()) } impl Vm { @@ -1660,12 +2110,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, + uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? + .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; - - #[cfg(CONFIG_DRM_ASAHI_G15)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -1691,6 +2141,7 @@ impl Vm { VmBinding { binding: None, bind_token: None, + generation: 0, active_users: 0, ttb: page_table.ttb(), }, @@ -1704,6 +2155,7 @@ impl Vm { } else { None }; + let bind_sequence = Arc::new(AtomicU64::new(0), GFP_KERNEL)?; let unique_gpuvm = gpuvm::GpuVm::new::( c"Asahi::GpuVm", &dev, @@ -1716,6 +2168,7 @@ impl Vm { va_range: va_range.clone(), page_table, mapping_epoch: mapping_epoch.clone(), + bind_sequence: bind_sequence.clone(), epoch: Some(0), mm, uat_inner, @@ -1725,11 +2178,14 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); + let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { + device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, mapping_epoch, + bind_sequence, lifetime: None, residency: None, context_bindings: None, @@ -1742,7 +2198,10 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); @@ -1753,15 +2212,18 @@ impl Vm { /// Attach a status and job lifetime tracking before sharing a newly created user VM with /// queues, without the G17 canonical context views, shared-object footprints and idle /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn with_tracked_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init( + crate::g17::status::VmStatus::new(), + GFP_KERNEL, + )?); self.lifetime = Some(lifetime::VmLifetime::new()?); Ok(self) } - /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). - #[cfg(CONFIG_DRM_ASAHI_G15)] + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it; G17 timeout + /// attribution compares and logs VMs by it). pub(crate) fn id(&self) -> u64 { self.id } @@ -1773,9 +2235,18 @@ impl Vm { /// and invalidated to the point of coherency first (never only invalidated: CPU-written lines /// may be dirty), so the copy sees what the GPU or a write-combined CPU view stored. The /// mappings cannot change during the copy (GPUVM exec lock). - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - let mut inner = lock_vm(&self.gpuvm, None, false)?; + self.read_lock()?.read_user(va, buf) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + pub(crate) fn read_lock(&self) -> Result { + Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -1845,9 +2316,7 @@ impl Vm { } pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return false; - }; + let mut inner = self.gpuvm.lock_inner(); Self::covers_range_locked(&mut inner, address, size, read, write) } @@ -1862,26 +2331,22 @@ impl Vm { if stride <= block_size { return false; } - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return false; - }; + let mut inner = self.gpuvm.lock_inner(); addresses.iter().all(|&address| { let Some(guard) = address.checked_add(block_size) else { return false; }; Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked( - &mut inner, guard, stride - block_size, false, false, - ) + && !Self::covers_range_locked(&mut inner, guard, stride - block_size, false, false) }) } - /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 + /// Checks all exact request tuples under one hold of the VM reservation. Access bit0 /// requests reading and bit1 writing. The epoch names this same snapshot. pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return (false, None); - }; + // Every mapping change holds the VM reservation, so readers need only that one. The full + // exec lock would also lock every external BO, making each submit O(external BOs). + let mut inner = self.gpuvm.lock_inner(); let epoch = inner.epoch; let covered = ranges.iter().all(|&(address, size, access)| { Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) @@ -1894,6 +2359,10 @@ impl Vm { self.mapping_epoch.as_ref()?.snapshot() } + fn current_bind_sequence(&self) -> u64 { + self.bind_sequence.load(Ordering::Acquire) + } + /// Get the translation table base for this Vm fn ttb(&self) -> u64 { self.binding.lock().ttb @@ -1929,6 +2398,49 @@ impl Vm { range: Range, prot: Prot, guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Best, + ) + } + + /// Lowest-address placement for a monotonic search across several VMs. + pub(crate) fn map_in_range_lowest( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + self.map_in_range_with_mode( + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Low, + ) + } + + #[allow(clippy::too_many_arguments)] + fn map_in_range_with_mode( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + mode: mm::InsertMode, ) -> Result { use kernel::drm::gem::BaseObject; if guard_size % UAT_PGSZ != 0 { @@ -1943,29 +2455,30 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } - let vm_bo = self - .gpuvm - .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; let size = object_range.range(); + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? + self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // A failed reservation drops its payload while exec still owns GEM @@ -1986,7 +2499,7 @@ impl Vm { 0, range.start, range.end, - mm::InsertMode::Best, + mode, ); let mut node = match result { Ok(node) => node, @@ -2004,9 +2517,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a GEM object into this Vm at a specific address. @@ -2024,33 +2544,32 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } - let vm_bo = self - .gpuvm - .obtain(&gem, pin_init!(VmBoData { segments <-new_mutex!(None) }))?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { self.gpuvm - .exec_lock(/*Some(&gem)*/ None, false) + .exec_lock_vm(/*Some(&gem)*/ None, false) .inspect_err(|_| { dev_err!(self.dev, "map_at(): Failed to exec_lock gpuvm\n"); })? }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - let mut vm_bo = Some(MappingBo::new(vm_bo, self.gpuvm.clone())); let uat_inner = inner.uat_inner.clone(); // Keep reservation-locking destructors out of the allocator payload @@ -2085,9 +2604,16 @@ impl Vm { let ret = inner.map_node(&node, prot); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + let retain = ret.is_err() + && inner.uat_inner.m4_adopted + && inner.uat_inner.failed.load(Ordering::Acquire); core::mem::drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Map a range of a GEM object into this Vm using GPUVM. @@ -2105,14 +2631,16 @@ impl Vm { let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; - // TODO: hack, populate the GEM object sg table first - { - let dev = unsafe { self.dev.as_bound() }; - let _sgt = crate::gem::held_sg_table(gem, dev)?; - } + // Pin and copy backing before taking BO or DMA reservation locks. + let dev: &device::Device = device.as_ref().as_ref(); + let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm - .obtain(gem, pin_init!(VmBoData { segments <- new_mutex!(None) }))?; + .obtain(gem, pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }))?; + prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; let mut resources = VmOpResources { preallocated_gpuvas: [ @@ -2125,19 +2653,13 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? + self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); - let mut vm_bo_guard = vm_bo.data().segments.lock(); - if vm_bo_guard.is_none() { - let dev = unsafe { self.dev.as_bound() }; - let segments = prefetch_sgt_segments(dev, gem)?; - vm_bo_guard.replace(segments); - } - core::mem::drop(vm_bo_guard); - // Preallocate the page tables, to fail early if we ENOMEM inner.page_table.alloc_pages(addr..(addr + size))?; @@ -2180,6 +2702,22 @@ impl Vm { vm_exec.unique_gpuvm.sm_map(req) } + /// Spend the binding generation only for an incarnation exposed to firmware. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn mark_binding_published(&self) -> Result { + let uat = self.gpuvm.lock_inner().uat_inner.clone(); + let binding = self.binding.lock(); + let slot = binding.binding.as_ref().ok_or(EIO)?.slot() as usize + UAT_USER_CTX_START; + let mut shared = uat.lock(); + if shared.owned_user_roots[slot] != tagged_root(binding.ttb, slot) + || shared.ttbs()[slot].ttb0.load(Ordering::Acquire) != shared.owned_user_roots[slot] + { + return Err(EIO); + } + shared.binding_published[slot] = true; + Ok(()) + } + /// Add a direct MMIO mapping to this Vm at a free address. pub(crate) fn map_io( &self, @@ -2189,7 +2727,9 @@ impl Vm { size: usize, prot: Prot, ) -> Result { - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( @@ -2223,22 +2763,35 @@ impl Vm { let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; + let transaction_uat = inner.uat_inner.clone(); + let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); - let ret = inner.page_table.map_pages( - iova..end, - phys as PhysicalAddr, - prot, - false, - ); + let ret = inner + .page_table + .map_pages(iova..end, phys as PhysicalAddr, prot, false); + let discard = if ret.is_err() { + inner.page_table.discard_partial_map(iova..end) + } else { + Ok(()) + }; + drop(handoff); if ret.is_err() { - inner.rollback_failed_map(iova, size as u64); + inner.finish_failed_map(iova, size); + } else { + inner.tlbi_contexts_naming_root(iova, size); + } + let retain = discard.is_err() && inner.uat_inner.m4_adopted; + if retain { + inner.uat_inner.failed.store(true, Ordering::Release); } - // Drop the exec_lock first, so that if map_node failed the - // KernelMappingInner destructur does not deadlock. drop(mutation); - let _ = inner; + drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Unmap everything in an address range. @@ -2280,7 +2833,9 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); let _commit = if user { @@ -2311,7 +2866,7 @@ impl Vm { let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), result => result?, }; @@ -2338,6 +2893,66 @@ impl Vm { Ok(()) } + fn drop_mappings_before(&self, gem: &gem::Object, sequence: u64) -> Result { + let _residency = self.enter_retirement_residency(); + // Removing whole mappings only does unmaps, so no preallocated VAs + // are needed for the indexed path. + let mut resources = VmOpResources { + preallocated_gpuvas: [None, None, None], + }; + let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; + + // The BO's newest sequence is checked under the same lock as binds. If + // it is not newer than the close, every mapping belongs to that close + // and the per-object list is safe and substantially cheaper to walk. + // SAFETY: See `lock_vm()`. + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { + Err(error) if error == ENOMEM => { + return self.drop_mappings_filtered(gem, sequence); + } + result => result?, + }; + let Some(bo) = vm_exec.unique_gpuvm.find_bo(gem) else { + return Ok(()); + }; + if bo.data().newest_bind.load(Ordering::Relaxed) > sequence { + drop(vm_exec); + drop(bo); + return self.drop_mappings_filtered(gem, sequence); + } + + mod_dev_dbg!(self.dev, "MMU: deferred bo_unmap\n"); + let result = vm_exec.unique_gpuvm.bo_unmap(&mut ctx, &bo); + if result.is_ok() { + self.untrack_context_object_before(gem, sequence); + self.untrack_shared_object_before(gem, sequence); + } + drop(vm_exec); + drop(bo); + if result == Err(ENOMEM) { + return self.drop_mappings_filtered(gem, sequence); + } + result + } + + /// A newer binding shares the BO list, so select the closed generation by + /// walking the VM without allocating. This is also the ENOMEM fallback. + fn drop_mappings_filtered(&self, gem: &gem::Object, sequence: u64) -> Result { + let mut inner = self.gpuvm.lock_inner(); + let result = inner.unmap_object_where( + gem, + |va| va.data().bind_sequence <= sequence, + VmInner::unmap_gpuva, + ); + if result.is_ok() { + self.untrack_context_object_before(gem, sequence); + self.untrack_shared_object_before(gem, sequence); + } + drop(inner); + self.bo_deferred_cleanup(); + result + } + /// Preserve the indexed fast path normally; memory pressure must not lose /// accepted object cleanup just because an exec or operation list failed. fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result { @@ -2384,8 +2999,18 @@ impl Drop for VmInner { let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.uat_inner.lock(); - uat_inner.handoff().lock(); + let mut transaction = match self.uat_inner.lock_handoff() { + Ok(guard) => guard, + Err(_) => { + self.page_table.retain(); + pr_err!( + "MMU: retaining M4 VM {} page tables after PPL failure\n", + self.id + ); + return; + } + }; + let uat_inner = transaction.shared_mut(); let handoff_cur = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); let inval = ttb_cur == ttb; @@ -2398,9 +3023,10 @@ impl Drop for VmInner { } uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); + uat_inner.owned_user_roots[idx] = 0; + uat_inner.publish_slot(idx); } - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + drop(transaction); // In principle we dropped all the KernelMappings already, but we might as // well play it safe and invalidate the whole ASID. @@ -2420,7 +3046,7 @@ impl Drop for VmInner { /// Physical regions of a UAT whose handoff the generation has already established itself /// ([`Uat::new_adopted`]). None of them needs to be in the kernel linear map. -#[cfg(CONFIG_DRM_ASAHI_G15)] +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) struct AdoptedRegions { /// The handoff region (PPL magic, lock and flush requests). pub(crate) handoff: Range, @@ -2431,34 +3057,56 @@ pub(crate) struct AdoptedRegions { } impl Uat { - #[cfg(CONFIG_DRM_ASAHI_G15)] + /// Common AP and firmware exclusion for G16's reserved-table window. Acquire + /// immutable root addresses before this guard, preserving binding -> UAT lock order. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn lock_m4_handoff(&self) -> Result> { + if !self.inner.m4_adopted { + return Err(EINVAL); + } + Ok(M4HandoffGuard { + _transaction: self.inner.lock_handoff()?, + }) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Map a region given by its physical range, write-combined. - fn map_physical(range: &Range, min_size: usize) -> Result { + fn map_physical( + range: &Range, + min_size: usize, + reserved_ram: bool, + ) -> Result { let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; - if size < min_size || (range.start | range.end) & (UAT_PGMSK as PhysicalAddr) != 0 { + if size < min_size || range.start & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); } // SAFETY: All-zero is a valid `struct resource` (no name, parent or children). let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as _; + raw.flags = kernel::bindings::IORESOURCE_MEM as usize; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; - // A carveout outside System RAM is mapped write-combined; a reserved-memory region - // inside System RAM (mapped by the kernel) through the kernel's own mapping, as - // `map_region` does for the other generations. - let mode = crate::g15::memory_mode(range.start, size as u64)?; - let ram = mode == kernel::bindings::MEMREMAP_WB as u32; - let flags = if ram { - io::mem::MemFlag::WB + let flags = io::mem::MemFlag::WC; + #[cfg(CONFIG_DRM_ASAHI_G15)] + let flags = if reserved_ram { + // G15 admits only whole reserved RAM pages or wholly non-RAM carveouts. + if crate::g15::memory_mode(range.start, size as u64)? + == kernel::bindings::MEMREMAP_WB as u32 + { + io::mem::MemFlag::WB + } else { + flags + } } else { - io::mem::MemFlag::WC + flags }; - // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU - // reaches them through its own translation. + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + let _ = reserved_ram; + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; + // the GPU reaches them through its own translation. let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, @@ -2466,7 +3114,7 @@ impl Uat { }) } - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Creates a `Uat` for a handoff that the caller establishes itself (G15). /// /// This only maps the handoff and the TTBAT (write-combined); neither is written here. The @@ -2480,23 +3128,52 @@ impl Uat { /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after /// mapping and before the firmware uses the new addresses. Page tables are never freed on /// unmap, so linked entries stay valid. + #[allow(dead_code)] pub(crate) fn new_adopted( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, ) -> Result { - let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; - let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; + Self::new_adopted_inner(dev, geometry, regions, true, true) + } + + /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also + /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; + /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn new_adopted_with_user_upper( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + map_kernel_to_user: bool, + ) -> Result { + Self::new_adopted_inner(dev, geometry, regions, map_kernel_to_user, false) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn new_adopted_inner( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + map_kernel_to_user: bool, + reserved_ram: bool, + ) -> Result { + let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE, reserved_ram)?; + let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE, reserved_ram)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { firmware: UatFirmware::Handoff, + m4_adopted: !map_kernel_to_user, + failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, + reserved: None, + quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -2505,10 +3182,14 @@ impl Uat { kernel_ttb1: regions.kernel_root, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, - map_kernel_to_user: true, + map_kernel_to_user, handoff_rgn, ttbs_rgn, + quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -2520,8 +3201,14 @@ impl Uat { let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; let shadow_root = kernel::page::Page::into_phys(shadow); - let kernel_lower_vm = - Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_lower_vm = Vm::new( + dev.into(), + inner.clone(), + geometry.user_range(), + geometry, + None, + 1, + )?; let kernel_vm = Vm::new( dev.into(), inner.clone(), @@ -2548,13 +3235,13 @@ impl Uat { }) } - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Root of the kernel lower `Vm`, for TTBAT context 0 of an adopted handoff. pub(crate) fn kernel_lower_root(&self) -> PhysicalAddr { self.kernel_lower_vm.ttb() } - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Root whose driver-window entry names the second-level table of [`Self::kernel_vm`]'s /// driver window. With [`Self::new_adopted`] this is the shadow root that no context names. pub(crate) fn kernel_window_root(&self) -> PhysicalAddr { @@ -2618,13 +3305,16 @@ impl Uat { return Err(EINVAL); } let root = { - let owner = lock_vm(&vm.gpuvm, None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { return Err(EINVAL); } owner.ttb() }; - Ok(self.inner.lock().contexts_naming_root(root)) + Ok(self + .inner + .lock() + .contexts_naming_root(root, self.inner.m4_adopted)) } /// Publish a completed alias set before its descriptors become visible. @@ -2653,12 +3343,24 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, true) + } + + /// Binds a VM without waiting for another slot owner to retire. + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn try_bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, false) + } + + fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { + self.inner.check_m4()?; let mut binding = vm.binding.lock(); + self.inner.check_m4()?; if binding.binding.is_none() { assert_eq!(binding.active_users, 0); - let isolation = *module_parameters::robust_isolation.value() != 0; + let isolation = module_parameters::robust_isolation.value() != 0; self.slots.set_limit(if isolation { NonZeroUsize::new(1) @@ -2666,22 +3368,34 @@ impl Uat { None }); - let slot = self.slots.get(binding.bind_token)?; + let slot = if wait { + self.slots.get(binding.bind_token)? + } else { + self.slots.try_get(binding.bind_token)? + }; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - let uat_inner = self.inner.lock(); - - let ttb1 = if uat_inner.map_kernel_to_user { - uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT + let mut transaction = self.inner.lock_handoff()?; + let uat_inner = transaction.shared_mut(); + // On M4, empty bind/close churn must not wrap the wire generation + // back to firmware's last observed identity for a different root. + let previous = uat_inner.binding_generations[idx]; + let incarnation = if !self.inner.m4_adopted + || previous == 0 + || uat_inner.binding_published[idx] + { + previous.checked_add(1).ok_or(EOVERFLOW)? } else { - 0 + previous }; + let ttb1 = + application_ttb1(uat_inner.kernel_ttb1, idx, uat_inner.map_kernel_to_user); + let ttbs = uat_inner.ttbs(); - uat_inner.handoff().lock(); if uat_inner.handoff().current_slot() == Some(idx as u32) { pr_err!( "Vm::bind to slot {}, but it is currently in use by the ASC?\n", @@ -2690,8 +3404,12 @@ impl Uat { } ttbs[idx].ttb0.store(ttb, Ordering::Release); ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); + uat_inner.binding_generations[idx] = incarnation; + uat_inner.binding_published[idx] = false; + uat_inner.owned_user_roots[idx] = ttb; + binding.generation = binding_wire_generation(incarnation); + uat_inner.publish_slot(idx); + drop(transaction); // Make sure all TLB entries from the previous owner of this ASID are gone mem::sync(); @@ -2710,6 +3428,35 @@ impl Uat { Ok(VmBind(vm.clone(), slot)) } + /// Confirms the exact stopped-adopted M4 TTBAT contract. The same mutex serializes every + /// host root installation/removal; firmware does not write these words. An unexpected + /// observed root is an error, never a new ownership grant. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) fn check_adopted_bindings(&self) -> Result { + self.inner.check_m4()?; + // Binding locks precede the global UAT lock, as in bind(). + let lower = self.kernel_lower_vm.ttb() | TTBR_VALID; + let inner = self.inner.lock(); + self.inner.check_m4()?; + if self.inner.firmware != UatFirmware::Handoff || inner.map_kernel_to_user { + return Err(EINVAL); + } + let ttbs = inner.ttbs(); + if ttbs[0].ttb0.load(Ordering::Acquire) != lower + || ttbs[0].ttb1.load(Ordering::Acquire) != inner.kernel_ttb1 | TTBR_VALID + { + return Err(EIO); + } + for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { + if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || table.ttb1.load(Ordering::Acquire) != 0 + { + return Err(EIO); + } + } + Ok(()) + } + /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { @@ -2717,7 +3464,7 @@ impl Uat { return Err(EINVAL); } { - let owner = lock_vm(&vm.gpuvm, None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) { @@ -2761,6 +3508,7 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -2770,14 +3518,17 @@ impl Uat { /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - Vm::new( + self.inner.check_m4()?; + let vm = Vm::new( self.dev.clone(), self.inner.clone(), kernel_range, self.geometry, None, id, - ) + )?; + self.inner.check_m4()?; + Ok(vm) } /// Returns the address space geometry of this UAT. @@ -2799,14 +3550,19 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + UAT_PGSZ / size_of::(), + |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }, + ) }?; mem::sync(); @@ -2819,11 +3575,16 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + snapshot.len(), + |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }, + ) }?; if !matches { return Err(EIO); @@ -2835,15 +3596,19 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let Some(secondary) = self.inner.lock().secondary_ttb1 else { + // A single-instance firmware has no secondary root to mirror. + return Ok(()); + }; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(secondary, count, |secondary| { + pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(reserved, secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -2860,24 +3625,33 @@ impl Uat { Ok(()) } + /// Discards the CPU's cached lines of the no-map table regions after the firmware built its + /// root there during boot (`UatQuirks::reserved_tables`; see `ReservedTables::invalidate`). + pub(crate) fn invalidate_reserved_tables(&self) { + if let Some(reserved) = self.inner.reserved.as_ref() { + reserved.invalidate(); + } + } + /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff().unlock(); + inner.handoff_unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); + inner.publish_slot(KERNEL_ALIAS_CTX); + inner.handoff_unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -2887,21 +3661,30 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { + fn make_inner( + dev: driver::AsahiDevice, + firmware: UatFirmware, + quirks: UatQuirks, + reserved: Option>, + ) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, - #[cfg(CONFIG_DRM_ASAHI_G15)] - clean_poc: false, + m4_adopted: false, + failed: AtomicBool::new(false), + clean_poc: quirks.clean_tables, + quirks, + reserved, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -2910,10 +3693,14 @@ impl Uat { kernel_ttb1: 0, execution_contexts: !EXECUTION_CONTEXT_MASK, context_generations: [0; UAT_NUM_CTX], + binding_generations: [0; UAT_NUM_CTX], + binding_published: [false; UAT_NUM_CTX], + owned_user_roots: [0; UAT_NUM_CTX], secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, + quirks: shared_quirks, }, "uat_shared" ), @@ -2929,12 +3716,27 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, + quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - let inner = Self::make_inner(dev.clone(), firmware)?; - + mem::set_page_tlbi(quirks.page_tlbi); let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let reserved = if quirks.reserved_tables { + let mut tables = pgtable::ReservedTables::new(); + for name in [c"pagetables", c"l2-pagetables"] { + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: The boot loader reserved these regions (no-map) for the firmware's + // page tables; they stay reserved until reboot. + unsafe { tables.add(res)? }; + } + Some(Arc::new(tables, GFP_KERNEL)?) + } else { + None + }; + + let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -2942,11 +3744,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset, - } => ( - secondary_root_offset as usize + UAT_PGSZ, - Some(ttb1 + secondary_root_offset), - ), + secondary_root_offset: Some(offset), + } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), + UatFirmware::PublishedRoots { + secondary_root_offset: None, + } => (UAT_PGSZ, None), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -2997,11 +3799,18 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware)?; + inner.handoff().init(firmware, quirks.handoff_slot)?; + if quirks.clean_tables { + pgtable::clean_poc( + core::ptr::from_ref(inner.handoff()).cast::(), + HANDOFF_SIZE, + ); + pgtable::poc_barrier(); + } dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); @@ -3013,7 +3822,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if secondary_ttb1.is_some() { + if matches!(firmware, UatFirmware::PublishedRoots { .. }) { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3022,8 +3831,12 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } + if quirks.clean_tables { + pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); + pgtable::poc_barrier(); + } - inner.handoff().unlock(); + inner.handoff_unlock(); core::mem::drop(inner); @@ -3041,3 +3854,30 @@ impl Drop for Uat { mem::sync(); } } + +impl Vm { + /// Diagnostic reservation occupancy (including guards), not a placement promise. + /// Removal can progress between queries; allocations are excluded by the VM lock. + pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { + let inner = self.gpuvm.lock_inner(); + let mut next = range.start; + let mut used = 0; + let mut largest = 0; + while next < range.end { + let Some(node) = inner.mm.first_overlap(next..range.end)? else { + largest = largest.max(range.end - next); + break; + }; + largest = largest.max(node.start.saturating_sub(next)); + let end = node.end.min(range.end); + used += end - node.start.max(next); + next = end; + } + Ok((used, largest)) + } +} +impl Uat { + pub(crate) fn execution_contexts_used(&self) -> u32 { + (self.inner.lock().execution_contexts & EXECUTION_CONTEXT_MASK).count_ones() + } +} diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 87ec3bd73658b..325babdecb370 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: (GPL-2.0-only OR MIT) AND GPL-2.0-only //! Support for Apple RTKit coprocessors. //! @@ -222,6 +222,7 @@ impl RtKit { } else { 0 }, + syslog_is_debug: None, crashed: Some(crashed_callback::), recv_message: Some(recv_message_callback::), recv_message_early: Some(recv_message_early_callback::), @@ -230,11 +231,18 @@ impl RtKit { } else { None }, + shmem_setup_ep: None, shmem_destroy: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { Some(shmem_destroy_callback::) } else { None }, + protocol_version: 0, + }; + + const COREANALYTICS_VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + flags: Self::VTABLE.flags | bindings::APPLE_RTKIT_NEEDS_COREANALYTICS, + ..Self::VTABLE }; /// Creates a new RTKit client for a given device and optional mailbox name or index. @@ -243,6 +251,29 @@ impl RtKit { mbox_name: Option<&'static CStr>, mbox_idx: usize, data: T::Data, + ) -> Result { + Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::VTABLE) + } + + /// Creates a client that requires CoreAnalytics setup during boot. + /// + /// The caller must qualify this dependency for the particular firmware + /// instance. This selects immutable operations before mailbox RX starts. + pub fn new_with_coreanalytics( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::COREANALYTICS_VTABLE) + } + + fn new_with_ops( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ops: &'static bindings::apple_rtkit_ops, ) -> Result { let ptr: *mut crate::ffi::c_void = data.into_foreign().cast(); let guard = ScopeGuard::new(|| { @@ -260,7 +291,7 @@ impl RtKit { None => ptr::null(), }, mbox_idx.try_into()?, - &Self::VTABLE, + ops, )) }?; @@ -274,6 +305,10 @@ impl RtKit { }) } + /// Borrows the C handle without transferring ownership. + /// The owner must outlive each unsafe C operation and its callbacks. + pub fn as_raw(&self) -> *mut bindings::apple_rtkit { self.rtk } + /// Requests acknowledged RTKit shutdown using the C core's bounded waits. /// /// The caller must not hold a lock needed by its receive callbacks: the @@ -334,6 +369,57 @@ impl RtKit { pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } } + + /// Sets the minimum utilization (0..=1024) the coprocessor's receive thread runs with, or + /// restores the scheduler's default with `None`. + /// + /// The receive thread applies it to itself when it next runs. Returns `EOPNOTSUPP` for a + /// mailbox without a threaded receive handler. + pub fn set_rx_util_min(&self, util_min: Option) -> Result { + // INVARIANT: `rtk` is valid per the type invariant. + RxUtilMin { rtk: self.rtk }.set(util_min) + } + + /// Returns a handle that sets the receive thread's minimum utilization (see + /// [`RtKit::set_rx_util_min`]) without borrowing this [`RtKit`], from any context. + /// + /// # Safety + /// + /// The caller must not use the handle after this [`RtKit`] is dropped. + pub unsafe fn rx_util_min_handle(&self) -> RxUtilMin { + // INVARIANT: The caller keeps this `RtKit`, and so `rtk`, alive while using the handle. + RxUtilMin { rtk: self.rtk } + } +} + +/// Sets the minimum utilization of an RTKit coprocessor's receive thread. +/// +/// The request is a single store that the receive thread applies to itself when it next runs, +/// so it may be made from any context. +/// +/// # Invariants +/// +/// `rtk` is valid while the handle is used. +#[derive(Clone, Copy)] +pub struct RxUtilMin { + rtk: *mut bindings::apple_rtkit, +} + +// SAFETY: The request is one store into the mailbox, valid from any thread. +unsafe impl Send for RxUtilMin {} +// SAFETY: As above; the handle has no state of its own. +unsafe impl Sync for RxUtilMin {} + +impl RxUtilMin { + /// See [`RtKit::set_rx_util_min`]. + pub fn set(&self, util_min: Option) -> Result { + let util_min = match util_min { + Some(v) => i32::try_from(v).map_err(|_| EINVAL)?, + None => -1, + }; + // SAFETY: `rtk` is valid per the type invariant; the call only stores the request. + to_result(unsafe { bindings::apple_rtkit_set_rx_util_min(self.rtk, util_min) }) + } } // SAFETY: `RtKit` operations require a mutable reference From 3f4f1a4ffa285145c29c66d428ecaff051449c76 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:03:32 -0500 Subject: [PATCH 29/37] drm/asahi: use the existing F32 bit accessor on 7.3 Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/float.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 51cac02624d3d..782cd47fc7111 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -44,11 +44,6 @@ struct F32U { } impl F32 { - /// Return the IEEE-754 binary32 representation. - pub(crate) const fn to_bits(self) -> u32 { - self.0 - } - /// Convert a raw 32-bit representation into an F32 pub(crate) const fn from_bits(u: u32) -> F32 { F32(u) From 40c0f578fa188d83dbe58f0cdd0780aff05696fa Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:13:17 -0500 Subject: [PATCH 30/37] selftests: asahi: match the 7.3 capability argument type Signed-off-by: Satya Benson --- tools/asahi/queue-priority-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/asahi/queue-priority-tests.py b/tools/asahi/queue-priority-tests.py index cfbbe8f36d4e6..1c5b4695c85f7 100644 --- a/tools/asahi/queue-priority-tests.py +++ b/tools/asahi/queue-priority-tests.py @@ -24,7 +24,7 @@ pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH:u32=2; } mod bindings { pub const CAP_SYS_NICE:u32=23; - pub unsafe fn capable(_:u32)->bool { CAP.with(|c|c.get()) } + pub unsafe fn capable(_:i32)->bool { CAP.with(|c|c.get()) } thread_local! {pub static CAP:std::cell::Cell=const {std::cell::Cell::new(false)};} } mod g17 {pub struct Gpu;} From 032b6040892ac4e56a3f21bd9966534de45be48c Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:13:18 -0500 Subject: [PATCH 31/37] selftests: asahi: accept explicitly registered test-prefix scripts Signed-off-by: Satya Benson --- tools/asahi/run-host-tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 5c5ca93c68594..53f803bddc8e5 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -188,7 +188,7 @@ def jobs(work, conf, dtc): if re.search(r'-tests\.(py|rs|sh)$', p.name) and p.name != Path(__file__).name} known = set(ASAHI_TESTS) | set(ASAHI_RUST) | set(ASAHI_SHELL) missing = sorted(found - known) - gone = sorted(known - found) + gone = sorted(name for name in known if not (ASAHI / name).is_file()) if missing or gone: for name in missing: print(f'not registered in {Path(__file__).name}: tools/asahi/{name}', file=sys.stderr) From 10e8e3a92db108eeb3c540dbfc3187a25002226d Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:24:03 -0500 Subject: [PATCH 32/37] drm/asahi: compose the carried timestamp adapter with Aurora 7.3 Keep Aurora current for all other backends. Use the carried public M3 firmware timestamp adapter and accept the new queue usage argument. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/asahi_core.rs | 86 +- drivers/gpu/drm/asahi/driver.rs | 256 ++-- drivers/gpu/drm/asahi/file.rs | 170 ++- drivers/gpu/drm/asahi/g15/command.rs | 43 +- drivers/gpu/drm/asahi/g15/mod.rs | 65 +- drivers/gpu/drm/asahi/g15/uat.rs | 12 +- drivers/gpu/drm/asahi/g17/buffer.rs | 16 + drivers/gpu/drm/asahi/g17/buffer/target.rs | 78 +- drivers/gpu/drm/asahi/g17/capacity.rs | 5 + drivers/gpu/drm/asahi/g17/completion.rs | 208 ++- drivers/gpu/drm/asahi/g17/context.rs | 200 ++- drivers/gpu/drm/asahi/g17/context/identity.rs | 256 +++- drivers/gpu/drm/asahi/g17/dependency.rs | 126 +- drivers/gpu/drm/asahi/g17/event.rs | 8 + drivers/gpu/drm/asahi/g17/feed.rs | 75 +- drivers/gpu/drm/asahi/g17/freelist.rs | 37 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 62 + drivers/gpu/drm/asahi/g17/fw/compute.rs | 8 +- .../gpu/drm/asahi/g17/fw/initdata/v25g83.rs | 108 +- drivers/gpu/drm/asahi/g17/fw/kick.rs | 14 + drivers/gpu/drm/asahi/g17/fw/queue.rs | 59 - drivers/gpu/drm/asahi/g17/fw/render.rs | 80 +- drivers/gpu/drm/asahi/g17/fw/variant.rs | 23 +- drivers/gpu/drm/asahi/g17/initdata.rs | 299 ++++- drivers/gpu/drm/asahi/g17/initdata/v25g83.rs | 44 +- drivers/gpu/drm/asahi/g17/job.rs | 76 +- drivers/gpu/drm/asahi/g17/kick.rs | 170 ++- drivers/gpu/drm/asahi/g17/mod.rs | 429 +++++- drivers/gpu/drm/asahi/g17/object.rs | 98 +- drivers/gpu/drm/asahi/g17/power.rs | 86 +- drivers/gpu/drm/asahi/g17/preparation.rs | 124 +- drivers/gpu/drm/asahi/g17/queue/admission.rs | 373 +++++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 906 +++++++++++-- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 29 +- .../drm/asahi/g17/queue/compute/retirement.rs | 463 ++++++- drivers/gpu/drm/asahi/g17/queue/mod.rs | 177 ++- drivers/gpu/drm/asahi/g17/queue/render.rs | 655 ++++++++- .../gpu/drm/asahi/g17/queue/render/memory.rs | 85 +- .../gpu/drm/asahi/g17/queue/render/parked.rs | 1 + .../drm/asahi/g17/queue/render/publication.rs | 10 +- .../drm/asahi/g17/queue/render/retirement.rs | 181 +++ drivers/gpu/drm/asahi/g17/recovery.rs | 200 ++- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 49 +- drivers/gpu/drm/asahi/g17/replay.rs | 67 +- drivers/gpu/drm/asahi/g17/runtime.rs | 1185 ++++++++++++++--- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 94 +- .../drm/asahi/g17/runtime/compute_control.rs | 13 +- .../drm/asahi/g17/runtime/compute_replay.rs | 24 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 149 ++- drivers/gpu/drm/asahi/g17/runtime/render.rs | 156 ++- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 6 +- .../drm/asahi/g17/runtime/render/observe.rs | 118 +- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 38 +- .../drm/asahi/g17/runtime/render/prepare.rs | 10 +- .../drm/asahi/g17/runtime/render/settle.rs | 14 + .../drm/asahi/g17/runtime/render/submit.rs | 434 +++++- .../drm/asahi/g17/runtime/render/teardown.rs | 48 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 23 +- drivers/gpu/drm/asahi/g17/teardown.rs | 148 ++ drivers/gpu/drm/asahi/g17/timeout.rs | 69 + drivers/gpu/drm/asahi/g17/validation.rs | 174 ++- drivers/gpu/drm/asahi/gem.rs | 71 +- drivers/gpu/drm/asahi/gpu.rs | 70 +- drivers/gpu/drm/asahi/hw/t6040.rs | 76 +- drivers/gpu/drm/asahi/hw/t8140.rs | 6 +- drivers/gpu/drm/asahi/m3/m3_drm.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_submit.rs | 9 +- drivers/gpu/drm/asahi/mem.rs | 6 +- drivers/gpu/drm/asahi/memacct.rs | 10 +- drivers/gpu/drm/asahi/mmu.rs | 418 ++++-- drivers/gpu/drm/asahi/mmu/bind.rs | 4 +- drivers/gpu/drm/asahi/mmu/m3.rs | 2 +- drivers/gpu/drm/asahi/mmu/residency.rs | 2 +- drivers/gpu/drm/asahi/object.rs | 42 +- drivers/gpu/drm/asahi/pgtable.rs | 62 +- drivers/gpu/drm/asahi/queue/mod.rs | 72 +- drivers/gpu/drm/asahi/regs.rs | 15 +- drivers/gpu/drm/asahi/sysfs.c | 94 +- drivers/gpu/drm/asahi/sysfs.h | 22 +- drivers/gpu/drm/asahi/sysfs_exports.rs | 79 +- drivers/gpu/drm/asahi/timestamp.rs | 45 +- drivers/gpu/drm/asahi/util.rs | 25 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 26 +- include/drm/drm_gem_shmem_helper.h | 4 + include/uapi/drm/asahi_drm.h | 93 +- rust/bindings/bindings_helper.h | 23 + rust/kernel/drm/device.rs | 25 +- rust/kernel/drm/file.rs | 19 +- 88 files changed, 8718 insertions(+), 1824 deletions(-) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 8b782a6f8ffa7..decf16ea09647 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -4,6 +4,7 @@ //! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. mod alloc; +mod binding; mod buffer; mod channel; mod cleanup; @@ -13,20 +14,20 @@ mod event; mod file; mod float; mod fw; -mod gem; -mod g17; +mod fwid; #[cfg(CONFIG_DRM_ASAHI_G15)] mod g15; -// Without the G15 backend, only the UAT adoption the T8122 runtime shares with it. -#[cfg(all(CONFIG_DRM_ASAHI_UAT_ADOPTION, not(CONFIG_DRM_ASAHI_G15)))] -#[path = "g15/adopt.rs"] -mod g15; +mod g17; +mod gem; + +#[cfg(CONFIG_DRM_ASAHI_G16)] +mod g16; mod gpu; mod hw; mod initdata; +mod m3; mod mem; mod memacct; -mod m3; mod microseq; mod mmu; mod object; @@ -35,8 +36,10 @@ mod queue; mod regs; mod slotalloc; mod stats; +mod stats_raw; mod sysfs_exports; mod timestamp; +mod usage; mod util; #[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] mod vm; @@ -48,12 +51,33 @@ use kernel::prelude::*; type G15Registration = kernel::driver::Registration>>; +#[cfg(CONFIG_DRM_ASAHI_G16)] +type G16Registration = kernel::driver::Registration>; +#[cfg(not(CONFIG_DRM_ASAHI_G16))] +type G16Registration = (); + +fn register_g16(module: &'static kernel::ThisModule) -> Result>>> { + #[cfg(CONFIG_DRM_ASAHI_G16)] + { + Ok(Some(KBox::pin_init( + kernel::driver::Registration::new(g16::DRIVER_NAME, module), + GFP_KERNEL, + )?)) + } + #[cfg(not(CONFIG_DRM_ASAHI_G16))] + { + let _ = module; + Ok(None) + } +} + #[pin_data] struct DriverModule { #[cfg(CONFIG_DRM_ASAHI_G15)] #[pin] _g15: G15Registration, // Field drop order unregisters/joins driver producers before queue teardown. + _g16: Option>>, #[pin] _driver: kernel::driver::Registration>, _cleanup: cleanup::QueueOwner, @@ -63,15 +87,18 @@ struct DriverModule { impl kernel::InPlaceModule for DriverModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit { + // SAFETY: Module init runs once, before either adapter is registered below. + unsafe { binding::init() }; try_pin_init!(Self { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, #[cfg(CONFIG_DRM_ASAHI_G15)] - _g15 <- kernel::driver::Registration::new(c"asahi-g15", module), + _g15 <- kernel::driver::Registration::new(c"aurora-g15", module), _driver <- kernel::driver::Registration::new( ::NAME, module, ), + _g16: register_g16(module)?, // All fallible fields are initialized. This final unit initializer cannot fail. #[cfg(CONFIG_DRM_ASAHI_G15)] _g15_ready: g15::activate(), @@ -81,9 +108,10 @@ impl kernel::InPlaceModule for DriverModule { module! { type: DriverModule, - name: "asahi", + name: "aurora", description: "AGX GPU driver for Apple silicon SoCs", license: "Dual MIT/GPL", + alias: ["asahi"], params: { debug_flags: u64 { default: 0, @@ -145,22 +173,6 @@ module! { default: 3072, description: "G15: PB growth maximum given to the firmware, 128 KiB blocks (64..4095)", }, - g15_pb_blocks: u32 { - default: 64, - description: "G15: PB blocks posted at (re)initialization, 128 KiB each (64..4095)", - }, - g15_pb_reclaim: u32 { - default: 2, - description: "G15: PB reclaim when idle: 0 never, 1 re-initialize, 2 also free backing", - }, - g15_pb_idle_ms: u32 { - default: 1000, - description: "G15: idle time before a PB reclaim, ms (at least 100)", - }, - g15_pb_reclaim_blocks: u32 { - default: 1024, - description: "G15: PB blocks posted above which an idle engine reclaims the PB", - }, g15_early_tiling: u32 { default: 0, description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", @@ -186,5 +198,29 @@ module! { default: 1, description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", }, + g16_deadline_ms: u32 { + default: 5000, + description: "G16: administrative completion-stamp deadline, ms (0 disables; long running work can exceed it)", + }, + g16_compute_deadline_ms: u32 { + default: 5000, + description: "G16: compute completion-stamp deadline, ms (0 permits unbounded compute; firmware recovery stays active)", + }, + g16_repoll_us: u32 { + default: 15, + description: "G16: repoll interval while publication waits for record consumption, us", + }, + g16_resnap_us: u32 { + default: 100, + description: "G16: completion repoll interval, us", + }, + g16_recovery: u32 { + default: 1, + description: "G16: answer qualified firmware recovery handshakes", + }, + g16_stream_gate: u32 { + default: 1, + description: "G16: validate CDM/VDM stream records", + }, }, } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 9b9b8fc4ccd15..6227bb4bf815b 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,11 +45,13 @@ pub(crate) struct AsahiPlatformDriver; // #[pin_data(PinnedDrop)] pub(crate) struct AsahiPlatformDriverData<'bound> { + binding: crate::binding::Owner, _device: ARef, _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, - /// The M3 T8122 runtime, stopped on unbind like the G17 GPU. + #[cfg(CONFIG_DRM_ASAHI_G16)] + pub(crate) g16: Option>, m3: Option, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. @@ -83,12 +85,18 @@ pub(crate) type AsahiDevRef = ARef; pub(crate) type AsahiRegData<'drm> = AsahiDrmRegistrationData<'drm>; /// DRM Driver metadata +/// +/// The major version names the UAPI generation in include/uapi/drm/asahi_drm.h +/// and only changes if that ABI ever breaks. The minor version goes up with +/// each new ioctl, flag or feature bit, so userspace can tell what a kernel +/// accepts without probing. The patch level marks fixes userspace may need to +/// detect. const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { - major: 0, + major: 1, minor: 0, patchlevel: 0, - name: c"asahi", - desc: c"Apple AGX Graphics", + name: c"aurora", + desc: c"Aurora GPU for Apple AGX", }; /// DRM Driver implementation for `AsahiDriver`. @@ -142,72 +150,82 @@ pub(crate) enum ProbeConfig { M3(crate::m3::Chip), } -// OF Device ID table. -kernel::of_device_table!( - OF_TABLE, - MODULE_OF_TABLE, - ::IdInfo, - [ - ( - of::DeviceId::new(c"apple,agx-t8140"), - ProbeConfig::G17(&hw::t8140::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6040"), - ProbeConfig::G17(&hw::t6040::CONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8103"), - ProbeConfig::Legacy(&hw::t8103::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t8112"), - ProbeConfig::Legacy(&hw::t8112::HWCONFIG) - ), - ( - of::DeviceId::new(c"apple,agx-t6000"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) - ), - ( - of::DeviceId::new(c"apple,agx-t6001"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) - ), - ( - of::DeviceId::new(c"apple,agx-t6002"), - ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) - ), - ( - of::DeviceId::new(c"apple,agx-t6020"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) - ), - ( - of::DeviceId::new(c"apple,agx-t6021"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) - ), - ( - of::DeviceId::new(c"apple,agx-t6022"), - ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) - ), - ( - of::DeviceId::new(c"apple,agx-t8122"), - ProbeConfig::M3(crate::m3::Chip::T8122) - ), - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - ( - of::DeviceId::new(c"apple,agx-t6030"), - ProbeConfig::M3(crate::m3::Chip::T6030) - ), - #[cfg(not(CONFIG_DRM_ASAHI_G15))] - ( - of::DeviceId::new(c"apple,agx-t6031"), - ProbeConfig::M3(crate::m3::Chip::T6031) - ), - ( - of::DeviceId::new(c"apple,agx-t6032"), - ProbeConfig::M3(crate::m3::Chip::T6032) - ), - ] +// OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on +// its entries, so the entries that the native G15 backend owns when it is +// built are passed in by configuration. +macro_rules! asahi_of_device_table { + ($($g15_owned:tt)*) => { + kernel::of_device_table!( + OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c"apple,agx-t8140"), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6040"), + ProbeConfig::G17(&hw::t6040::CONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8103"), + ProbeConfig::Legacy(&hw::t8103::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t8112"), + ProbeConfig::Legacy(&hw::t8112::HWCONFIG) + ), + ( + of::DeviceId::new(c"apple,agx-t6000"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6000) + ), + ( + of::DeviceId::new(c"apple,agx-t6001"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6001) + ), + ( + of::DeviceId::new(c"apple,agx-t6002"), + ProbeConfig::Legacy(&hw::t600x::HWCONFIG_T6002) + ), + ( + of::DeviceId::new(c"apple,agx-t6020"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6020) + ), + ( + of::DeviceId::new(c"apple,agx-t6021"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6021) + ), + ( + of::DeviceId::new(c"apple,agx-t6022"), + ProbeConfig::Legacy(&hw::t602x::HWCONFIG_T6022) + ), + ( + of::DeviceId::new(c"apple,agx-t8122"), + ProbeConfig::M3(crate::m3::Chip::T8122) + ), + $($g15_owned)* + ( + of::DeviceId::new(c"apple,agx-t6032"), + ProbeConfig::M3(crate::m3::Chip::T6032) + ), + ] + ); + }; +} + +#[cfg(not(CONFIG_DRM_ASAHI_G15))] +asahi_of_device_table!( + ( + of::DeviceId::new(c"apple,agx-t6030"), + ProbeConfig::M3(crate::m3::Chip::T6030) + ), + ( + of::DeviceId::new(c"apple,agx-t6031"), + ProbeConfig::M3(crate::m3::Chip::T6031) + ), ); +#[cfg(CONFIG_DRM_ASAHI_G15)] +asahi_of_device_table!(); /// Platform Driver implementation for `AsahiDriver`. impl platform::Driver for AsahiPlatformDriver { @@ -215,19 +233,16 @@ impl platform::Driver for AsahiPlatformDriver { const OF_ID_TABLE: Option> = Some(&OF_TABLE); type Data<'bound> = AsahiPlatformDriverData<'bound>; - fn unbind<'bound>( - _pdev: &'bound platform::Device>, - this: Pin<&Self::Data<'bound>>, - ) { + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { if let Some(gpu) = this.g17.as_ref() { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] if let Some(m3) = this.m3.as_ref() { - // As for G17: the runtime stops its coprocessor through device resources. m3.stop(); } + this.close_binding(); } /// Device probe function. @@ -248,6 +263,7 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; + // SAFETY: The selected SoC's UAT output width bounds all device DMA. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; @@ -264,6 +280,9 @@ impl platform::Driver for AsahiPlatformDriver { .required_by(pdev.as_ref())?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { (hw::GpuGen::G13, _, &[12, 3, 0]) => { @@ -295,12 +314,20 @@ impl platform::Driver for AsahiPlatformDriver { gpu.init()?; let stats = gpu.stats_snapshot(); + let debug_stats = gpu.raw_stats_snapshot(); let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is // unbound; it is never forgotten. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + // The DRM core created this card's debugfs directory when the device was registered; + // publish the raw firmware statistics in it. drm_dev_unregister() removes the file, and + // waits for its readers, before the registration data that owns the snapshot is dropped. + if let Some(snapshot) = debug_stats.as_ref() { + crate::stats_raw::register(reg.device(), snapshot); + } + // Register the sysfs file on the platform device. Must happen after // the DRM device is registered so the device is fully bound. let raw_dev = pdev.as_ref().as_raw(); @@ -308,14 +335,17 @@ impl platform::Driver for AsahiPlatformDriver { let raw_stats = crate::sysfs_exports::register( raw_dev, snapshot, - stats.is_some() && *module_parameters::stats_export.value() != 0, + stats.is_some() && module_parameters::stats_export.value() != 0, )?; let driver = AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, raw_dev, raw_stats, stats, @@ -335,6 +365,9 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -344,10 +377,13 @@ impl AsahiPlatformDriver { // G15 retains its firmware-visible state independently until reboot, even on error. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -366,6 +402,7 @@ impl AsahiPlatformDriver { unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; let resources = regs::Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?; let gpu = backend.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); @@ -380,6 +417,9 @@ impl AsahiPlatformDriver { }; dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); Ok(AsahiPlatformDriverData { + binding, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, _device: reg.device().into(), _reg: reg, g17: None, @@ -399,6 +439,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiDrmRegistrationData { @@ -414,9 +457,14 @@ impl AsahiPlatformDriver { return Err(error); } }; - // G17's firmware channel format has no supported stats decoder yet. + let stats = gpu::Gpu::stats_snapshot(&*gpu); let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register(raw_dev, core::ptr::null(), false) { + let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); + let raw_stats = match crate::sysfs_exports::register_g17( + raw_dev, + snapshot, + module_parameters::stats_export.value() != 0, + ) { Ok(handle) => handle, Err(error) => { gpu.shutdown(); @@ -426,13 +474,58 @@ impl AsahiPlatformDriver { if let Err(error) = crate::sysfs_exports::register_idle(raw_dev, crate::g17::power::IDLE_HOLD_MS.as_ptr()) { - dev_warn!(pdev.as_ref(), "idle_off_delay_ms not registered: {:?}\n", error); + dev_warn!( + pdev.as_ref(), + "idle_off_delay_ms not registered: {:?}\n", + error + ); } Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: Some(gpu), m3: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, + raw_dev, + raw_stats, + stats, + }) + } +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl AsahiPlatformDriver { + pub(crate) fn probe_g16<'bound>( + pdev: &'bound platform::Device>, + cfg: &'static crate::g16::Config, + ) -> Result> { + // SAFETY: the qualified UAT output width constrains this device's DMA. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + let resources = regs::Resources::new(pdev)?; + let res = regs::G16Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; + let gpu = crate::g16::Gpu::new(pdev, &drm, cfg, res)?; + let data_gpu = gpu.clone() as Arc; + let data = try_pin_init!(AsahiDrmRegistrationData { + gpu: data_gpu, + resources, + }); + // SAFETY: registration is stored in bound platform data and never forgotten. + let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + let raw_dev = pdev.as_ref().as_raw(); + let raw_stats = crate::sysfs_exports::register(raw_dev, core::ptr::null(), false)?; + Ok(AsahiPlatformDriverData { + binding, + _device: reg.device().into(), + _reg: reg, + g17: None, + m3: None, + g16: Some(gpu), raw_dev, raw_stats, stats: None, @@ -440,8 +533,15 @@ impl AsahiPlatformDriver { } } +impl AsahiPlatformDriverData<'_> { + pub(crate) fn close_binding(&self) { + self.binding.close(); + } +} + impl Drop for AsahiPlatformDriverData<'_> { fn drop(&mut self) { + self.binding.close(); // `self.raw_dev` was set in `probe` from the platform // device's `as_raw()`, valid for the lifetime of the bound driver. if !self.raw_dev.is_null() { diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index c627bd9c6b8a0..24e8db30e580e 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -16,8 +16,8 @@ use crate::{ AsahiDrmDevice, // }, gem, - memacct, gpu, + memacct, mmu, queue, util::{ @@ -127,10 +127,23 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, + pub(crate) cumulative: Option, + pub(crate) timeline: bool, } impl SyncItem { - fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + fn parse_one( + file: &DrmFile, + mut data: uapi::drm_asahi_sync, + out: bool, + bundled: bool, + ) -> Result { + if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !out || !bundled || data.timeline_value == 0 { + return Err(EINVAL); + } + data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; + } match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -151,6 +164,8 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -173,12 +188,10 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: if out { - Some(dma_fence::FenceChain::new()?) - } else { - None - }, + chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: true, }) } _ => { @@ -193,28 +206,52 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, + allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let mut anchor = None; - // SAFETY: We only read this once, so there are no TOCTOU issues. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); + // Read each record once so validation and use see the same retained bytes. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as usize), size).reader(); for i in 0..count { let mut sync: MaybeUninit = MaybeUninit::uninit(); // SAFETY: The size of `sync` is STRIDE reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) + core::slice::from_raw_parts_mut(sync.as_mut_ptr().cast::>(), STRIDE) })?; // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !allow_cumulative || i < in_count || anchor.is_some() { + return Err(EINVAL); + } + anchor = Some(i as usize); + } + vec.push( + Self::parse_one(file, sync, i >= in_count, allow_cumulative)?, + GFP_KERNEL, + )?; + } + if let Some(anchor) = anchor { + let bundle = crate::g17::sync_output::SignalBundle::new( + &vec[in_count as usize..], + anchor - in_count as usize, + )?; + vec[anchor].cumulative = Some(bundle); + } else { + for sync in &mut vec[in_count as usize..] { + if sync.timeline { + sync.chain_fence = Some(dma_fence::FenceChain::new()?); + } + } } Ok(vec) @@ -238,11 +275,13 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, + /// GPU memory this client created and holds. + mem: Arc, + /// Completed engine time, retained by queues after handle destruction. + usage: Arc, /// Largest size this file has asked GET_PARAMS for the global parameters with /// ([`gpu::Gpu::admit_vm`]). params_size: core::sync::atomic::AtomicU64, - /// GPU memory this client created and holds. - mem: Arc, } /// Convenience type alias for our DRM `File` type. @@ -263,9 +302,10 @@ impl drm::file::DriverFile for File { let id = gpu.ids().file.next(); let user_range = gpu.user_range(); let mem = memacct::Client::new()?; + let usage = Arc::new(crate::usage::Client::new(), GFP_KERNEL)?; mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - KBox::pin_init(File::new(id, user_range, mem), GFP_KERNEL) + KBox::pin_init(File::new(id, user_range, mem, usage), GFP_KERNEL) }) } @@ -279,6 +319,17 @@ impl drm::file::DriverFile for File { }) } + fn engine_stats(&self, mut emit: impl FnMut(&CStr, u64)) { + let names = [ + kernel::c_str!("vertex"), + kernel::c_str!("fragment"), + kernel::c_str!("compute"), + ]; + for (name, ns) in names.into_iter().zip(self.usage.snapshot()) { + emit(name, ns); + } + } + fn as_raw(&self) -> *mut bindings::drm_file { todo!() } @@ -292,17 +343,21 @@ impl File { id: u64, user_range: Range, mem: Arc, + usage: Arc, ) -> impl PinInit { + // SAFETY: The initializer supplies aligned, pinned storage for Self. + // XArray::new is infallible; each pinned array is initialized in place, + // then every remaining field is written before returning success. unsafe { pin_init::pin_init_from_closure(move |slot: *mut Self| { let raw_vms = addr_of_mut!((*slot).vms); - crate::util::raw_try_init( + pin_init::raw_try_init( raw_vms, xarray::XArray::>::new(xarray::AllocKind::Alloc1), )?; let raw_queues = addr_of_mut!((*slot).queues); - crate::util::raw_try_init( + pin_init::raw_try_init( raw_queues, xarray::XArray::>>>::new( xarray::AllocKind::Alloc1, @@ -310,15 +365,16 @@ impl File { )?; let raw_objects = addr_of_mut!((*slot).objects); - crate::util::raw_try_init( + pin_init::raw_try_init( raw_objects, xarray::XArray::>::new(xarray::AllocKind::Alloc1), )?; (*slot).id = id; (*slot).user_range = user_range; - addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); addr_of_mut!((*slot).mem).write(mem); + addr_of_mut!((*slot).usage).write(usage); + addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); Ok(()) }) } @@ -413,13 +469,13 @@ impl File { let size = core::mem::size_of::().min(data.size.try_into()?); - // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + // Write the retained result to this user slice once. let mut params_writer = UserSlice::new(UserPtr::from_addr(data.pointer as usize), size).writer(); // SAFETY: `size` is at most the sizeof of `params` params_writer.write_slice(unsafe { - core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) + core::slice::from_raw_parts(core::ptr::from_ref(¶ms).cast::(), size) })?; file.inner() @@ -632,7 +688,7 @@ impl File { /// IOCTL: vm_destroy: Destroy a `Vm`. pub(crate) fn vm_destroy( _device: &AsahiDrmDevice, - reg_data: &driver::AsahiDrmRegistrationData<'_>, + _reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_vm_destroy, file: &DrmFile, ) -> Result { @@ -641,7 +697,6 @@ impl File { Err(ENOENT) } else { drop(vm); - reg_data.gpu.vm_destroyed(); Ok(0) } } @@ -685,26 +740,47 @@ impl File { }; let size = gem::checked_object_size(data.size.try_into()?)?; - let commit = memacct::LEDGER.admit(&file.inner().mem, size as u64).inspect_err(|_| { - cls_pr_debug!( - Errors, - "gem_create: {:#x} bytes refused by the GPU memory limits\n", - size - ); - })?; + let commit = memacct::LEDGER + .admit(&file.inner().mem, size as u64) + .inspect_err(|_| { + cls_pr_debug!( + Errors, + "gem_create: {:#x} bytes refused by the GPU memory limits\n", + size + ); + })?; // Large G17 user objects are populated in full by the first VM_BIND. // Do not multiply direct reclaim/compaction latency by their folio // count. Userspace can evict its BO cache and retry after ENOMEM. - let pages_no_reclaim = gem::user_pages_no_reclaim( - size, - reg_data.gpu.as_any().is::(), - ); + let pages_no_reclaim = + gem::user_pages_no_reclaim(size, reg_data.gpu.as_any().is::()); let gem = gem::new_object( - device, size, data.flags, resv_obj, reclaim_cpu, pages_no_reclaim, commit, + device, + size, + data.flags, + resv_obj, + reclaim_cpu, + pages_no_reclaim, + commit, )?; if reg_data.gpu.prepare_new_objects() { - gem::prepare_new_object(&gem)?; + #[cfg(CONFIG_DRM_ASAHI_G15)] + if reg_data.gpu.as_any().is::() { + gem::prepare_new_object_g15(&gem)?; + } else { + let registered = device.registration_guard().ok_or(ENODEV)?; + let parent: &kernel::platform::Device = registered.as_ref(); + gem::prepare_new_object(&gem, parent.as_ref())?; + } + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + let registered = device.registration_guard().ok_or(ENODEV)?; + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + { + let parent: &kernel::platform::Device = registered.as_ref(); + gem::prepare_new_object(&gem, parent.as_ref())?; + } } + let handle = gem.create_handle(file)?; data.handle = handle; @@ -1278,15 +1354,21 @@ impl File { if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH && !reg_data.gpu.as_any().is::() { - cls_pr_debug!(Errors, "queue_create: HIGH priority is not supported on this GPU\n"); + cls_pr_debug!( + Errors, + "queue_create: HIGH priority is not supported on this GPU\n" + ); return Err(EINVAL); } - // SAFETY: capable() only reads the calling task's credentials (ioctl process context). if data.priority == uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH - && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as _) } + // SAFETY: capable() reads the current ioctl task's credentials. + && !unsafe { bindings::capable(bindings::CAP_SYS_NICE as i32) } && !file.is_current_master() { - cls_pr_debug!(Errors, "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n"); + cls_pr_debug!( + Errors, + "queue_create: HIGH priority needs CAP_SYS_NICE or DRM master\n" + ); return Err(EACCES); } @@ -1309,6 +1391,7 @@ impl File { // TODO: Plumb deeper the enum uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, data.usc_exec_base, + file.inner().usage.clone(), )?; queue.reserve_render_scs(data.render_scs_layout)?; @@ -1397,8 +1480,13 @@ impl File { if guarded { copy_commands(&mut vec)?; } - let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + let syncs = SyncItem::parse_array( + file, + data.syncs, + data.in_sync_count, + data.out_sync_count, + gpu.as_any().is::(), + )?; mod_dev_dbg!( device, diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs index b343aa3fe4e5b..66e52e6981197 100644 --- a/drivers/gpu/drm/asahi/g15/command.rs +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -146,6 +146,11 @@ pub(crate) struct Context<'a> { pub(crate) usc_base: u64, } +struct CheckedContext<'a> { + vm: &'a mut mmu::VmReadGuard, + usc_base: u64, +} + /// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are /// checked ([`Prepared::validate`]). pub(crate) struct Prepared<'a> { @@ -360,8 +365,8 @@ impl Timestamps<'_> { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(buffer) = object; - let mapping = buffer.firmware_mapping()?; + let file::Object::TimestampBuffer(mapping) = object; + let mapping = mapping.firmware_mapping()?; let offset = timestamp.offset as usize; if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { return Err(EINVAL); @@ -377,7 +382,13 @@ impl Timestamps<'_> { /// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested /// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a /// page without the access EACCES. -pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result { +pub(crate) fn cover( + vm: &mut mmu::VmReadGuard, + start: u64, + end: u64, + read: bool, + write: bool, +) -> Result { let mask = mmu::UAT_PGMSK as u64; if end <= start { return Err(EINVAL); @@ -407,7 +418,11 @@ pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) } /// Attachment spans, 128-byte granular, actually mapped with `write` or read access. -fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result { +fn check_attachments( + vm: &mut mmu::VmReadGuard, + list: &[uapi::drm_asahi_attachment], + write: bool, +) -> Result { for attachment in list { let end = attachment .pointer @@ -467,8 +482,8 @@ fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) { } /// Per-command render admission. -fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { - let vm = context.vm; +fn check_render(context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { + let vm = &mut *context.vm; let allowed = PROCESS_EMPTY_TILES | NO_VERTEX_CLUSTERING | RSRC_SPEC_HI @@ -552,11 +567,11 @@ fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Resu /// Per-command compute admission. fn check_compute( - context: &Context<'_>, + context: &mut CheckedContext<'_>, cmd: &uapi::drm_asahi_cmd_compute, attachments: &[uapi::drm_asahi_attachment], ) -> Result { - let vm = context.vm; + let vm = &mut *context.vm; let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end); if cmd.flags != 0 || cmd.sampler_count != 0 @@ -650,6 +665,12 @@ impl Prepared<'_> { counts, timestamps, } = self; + // On error, unlock before the last timestamp mapping can drop and unmap. + let mut access = context.vm.read_lock()?; + let mut context = CheckedContext { + vm: &mut access, + usc_base: context.usc_base, + }; let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?; let mut compute = KVec::new(); let mut index = 0; @@ -658,12 +679,12 @@ impl Prepared<'_> { match entry.kind { SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => { // Validated readable, never serialized on G15. - check_attachments(context.vm, &attachment_list(entry.payload)?, false)?; + check_attachments(&mut *context.vm, &attachment_list(entry.payload)?, false)?; } SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?, CMD_RENDER => { let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; - check_render(context, &cmd)?; + check_render(&mut context, &cmd)?; commands.push( Command::Render(KBox::new( RenderCommand { @@ -678,7 +699,7 @@ impl Prepared<'_> { } CMD_COMPUTE => { let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; - check_compute(context, &cmd, &compute)?; + check_compute(&mut context, &cmd, &compute)?; let slots = resolved.get(index).ok_or(EINVAL)?; commands.push( Command::Compute(KBox::new( diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 48e6afa948c98..ae0512de0b156 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -14,9 +14,9 @@ //! //! # Plugging into the shared driver //! -//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait -//! for the native display and PMP supplier before booting firmware. T6031 retains explicit -//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Driver`] is a separate platform driver with a module alias. Its probe must wait for +//! the native display (T6030 and T6031) and the PMP supplier (T6030) before booting +//! firmware. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. //! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), //! queues (`new_queue`, see [`queue`]) and timestamp buffers. //! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked @@ -45,6 +45,7 @@ mod adt; mod board; mod boot; +mod retain; // One admission rule for the generation mappings and shared adopted UAT. pub(crate) use boot::glue::memory_mode; @@ -143,22 +144,23 @@ const OF_TABLE: kernel::device_id::IdArray = kernel::device_id::IdArray::new([ ( of::DeviceId::new(c_str!("apple,agx-t6030")), - &hw::t6030::CONFIG, + &&hw::t6030::CONFIG, ), ( of::DeviceId::new(c_str!("apple,agx-t6031")), - &hw::t6031::CONFIG, + &&hw::t6031::CONFIG, ), ]); -// T6030 probe admission requires the native display and PMP supplier before -// firmware starts. T6031 retains explicit loading until it has an equivalent -// display-order admission gate. +// Probe admission requires the native display before firmware starts (T6030 +// also its PMP supplier), so both may be loaded at device creation. kernel::of_device_table!( AUTOLOAD_OF_TABLE, - MODULE_OF_TABLE_G15, - (), - [(of::DeviceId::new(c_str!("apple,agx-t6030")), ())] + @none, + [ + of::DeviceId::new(c_str!("apple,agx-t6030")), + of::DeviceId::new(c_str!("apple,agx-t6031")), + ] ); /// The G15 platform adapter: the shared DRM registration probed through @@ -177,6 +179,12 @@ impl platform::Driver for Driver<'_> { type Data<'bound> = Driver<'bound>; const OF_ID_TABLE: Option> = Some(&OF_TABLE); + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + if let Some(driver) = &this._driver { + driver.close_binding(); + } + } + /// The firmware session cannot be stopped once started. fn suppress_bind_attrs() -> bool { true @@ -211,7 +219,6 @@ impl platform::Driver for Driver<'_> { }, }) } - } /// Firmware starts only after every fallible module/driver registration has succeeded. @@ -374,7 +381,7 @@ pub(crate) struct Firmware { pub(crate) recovery: recovery::Recovery, /// Render UMA/PB allocators, their spare set and parameter-buffer growth. pub(crate) pools: pools::Pools, - /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. + /// Live VMs and the client aliases of every user VM. pub(crate) vms: Arc, } @@ -439,7 +446,7 @@ impl Gpu { *shared.state.lock() = Some(firmware); // The engine's statistics file (a debugfs failure does not fail the probe). if let Err(error) = shared.start_engine() { - dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); + dev_warn!(dev.as_ref(), "G15: no aurora-g15 debugfs ({:?})\n", error); } let gpu = Arc::new( Gpu { @@ -452,17 +459,18 @@ impl Gpu { }, GFP_KERNEL, )?; - gpu.retain(); + gpu.retain()?; Ok(gpu) } /// Keeps the firmware session and this module alive until reboot. /// - /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the + /// Takes the module reference once; `boot::start` already took it before touching the + /// firmware, so this only confirms it. Nothing firmware-reachable is freed on any path: the /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a /// failed start) until reboot. - pub(crate) fn retain(&self) { - boot::glue::retain_module(); + pub(crate) fn retain(&self) -> Result { + boot::glue::retain_module() } } @@ -537,7 +545,21 @@ impl Firmware { vms, }, ); - if let Err(e) = firmware.init_graphs() { + let result = firmware.init_graphs().and_then(|_| { + if cfg.platform != hw::Platform::AdtWithPmp { + return Ok(()); + } + // A report before firmware startup is ACKed without enabling bandwidth scaling. + // Reassert it after setup, including the backend's final device control. + // SAFETY: This session retains its acknowledged T6030 AGX power vote. + let ret = unsafe { boot::glue::asahi_g15_pmp_ready() }; + if ret == 0 { + Ok(()) + } else { + Err(Error::from_errno(ret)) + } + }); + if let Err(e) = result { let Firmware { session, init, @@ -632,7 +654,7 @@ impl gpu::Gpu for Gpu { Some(params::MAX_BINDS) } - fn vm_destroyed(&self) { + fn vm_closed(&self, _vm: &mmu::Vm) { vm::closed(self) } @@ -647,9 +669,10 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { Ok(KBox::new( - queue::Queue::new(self, vm, priority, usc_exec_base)?, + queue::Queue::new(self, vm, priority, usc_exec_base, usage)?, GFP_KERNEL, )?) } diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 1d63fb83b9fa1..0f91f7c9f0610 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -25,8 +25,10 @@ //! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these //! redundant but harmless.) //! -//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` -//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! Jobs acquire exclusive TTBAT slots without waiting at publication (`mmu::Uat::try_bind`) +//! and return them after successful retirement. Failed published jobs retain their slots until +//! their last reference drops (until reboot for jobs retained after a fail-stop). +//! The shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | //! slot << 48)` under the PPL lock and invalidates the slot's ASID. use core::ops::Range; @@ -36,14 +38,12 @@ use kernel::{ bindings, device, io::Io, prelude::*, + sync::Arc, time::{delay::fsleep, Delta, Instant, Monotonic}, }; -#[cfg(CONFIG_DRM_ASAHI_G15)] -use kernel::sync::Arc; use super::hw; use super::regs::{mb, rmb, wmb, Mapping}; -#[cfg(CONFIG_DRM_ASAHI_G15)] use crate::driver::AsahiDrmDevice as AsahiDevice; use crate::{mmu, pgtable}; @@ -69,7 +69,6 @@ const fn indices(va: u64) -> [usize; 3] { } /// Creates the adopted UAT over the ADT regions (step 2 above). -#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn adopt( dev: &AsahiDevice, cfg: &'static super::Config, @@ -202,7 +201,6 @@ pub(crate) struct Window { impl Window { /// Maps the UAT handoff regions ([`Mapping::region`]). - #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn new(adt: &super::adt::GpuAdt) -> Result { Self::from_regions(adt.handoff.start, adt.ttbs.start, adt.tables.start, adt.shared_l2.start) } diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 8f3a03fbb8dba..fd6460db2d1fb 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -677,6 +677,22 @@ impl Manager { pub(crate) fn blocks(&self) -> usize { self.tvb.blocks() } + /// Read-only diagnostic while firmware is halted. In particular this + /// does not reset the high-water counter as automatic_target() does. + pub(crate) fn report_recovery(&self) { + let read = |offset| -> Result { + Ok(self.graph.word(offset)?.load(Ordering::Relaxed)) + }; + pr_warn!("G17: recovery tvb id={} blocks={} cursors={:?} pages={:?} capacity={:?} producer={:?} consumer={:?} max_pages={:?} reset={:?} initialized={}\n", + self.buffer_id, self.blocks(), self.cursors(), + read(core::mem::offset_of!(fw::State, pages)), + read(core::mem::offset_of!(fw::State, block_capacity)), + read(core::mem::offset_of!(fw::State, producer)), + read(core::mem::offset_of!(fw::State, consumer)), + read(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages)), + read(fw::CONTROL + core::mem::offset_of!(fw::Control, reset)), + self.initialized); + } pub(crate) fn state_va(&self) -> u64 { self.graph.gpu_va() } diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index 64c88da8eca14..bd96ecc9d460e 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -29,7 +29,12 @@ fn aligned(value: usize, alignment: usize) -> Result { /// the default parameter-buffer sizes. Sizes are /// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; -const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, 0x600, 0x80]; +const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, GTP6_SIZE, 0x80]; +/// Room for the tiler's records across a partial render. macOS uses 0x1c00, so this is a superset, +/// and the size every partial-render test ran with. +const GTP_SLOTS: usize = 64; +const GTP6_SIZE: usize = GTP_SLOTS + * (((cfg::TVB_MAX_BLOCKS * cfg::TVB_BLOCK_SIZE).div_ceil(0x8000) + 0x17ff) / 24 * 8 + 8); /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] @@ -139,6 +144,16 @@ impl Layout { }) } pub(crate) fn tpc_capacity(&self) -> Result { + // TPC_MAX bounds the tail-pointer cache itself. On 25G83 this object + // also contains GTP sub-buffers, notably 64 times the tile map. Applying + // the cache limit to their combined extent rejects valid large targets. + // Keep the original cache and tile-record limits; they also bound the + // derived auxiliary extent before any backing allocation or replacement. + if aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN) > cfg::TPC_MAX + || self.record_stride()? > cfg::TARGET_RECORD_MAX + { + return Err(ERANGE); + } Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { @@ -188,7 +203,7 @@ impl Target { pub(crate) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { let tpc = layout.tpc_capacity()?; let stride = layout.record_stride()?; - if tpc > cfg::TPC_MAX || stride > cfg::TARGET_RECORD_MAX { + if stride > cfg::TARGET_RECORD_MAX { return Err(ERANGE); } let records = Object::new(dev, vm, stride * DEPTH)?; @@ -272,9 +287,6 @@ impl Target { let stride = layout.record_stride()?; let mut grew = false; if tpc > self.tpc.backing.size() { - if tpc > cfg::TPC_MAX { - return Err(ERANGE); - } Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; grew = true; } @@ -350,3 +362,59 @@ impl Target { } } } + + +#[kernel::macros::kunit_tests(asahi_g17_target_capacity)] +mod tests { + use super::*; + + fn pass(size: u32, utile: u8) -> RenderPass { + // SAFETY: RenderPass and its nested surfaces/programs contain only + // integers and bools, for which the all-zero representation is valid. + let mut pass: RenderPass = unsafe { core::mem::zeroed() }; + pass.width = size; + pass.height = size; + pass.layers = 1; + pass.utile_width = utile; + pass.utile_height = utile; + pass + } + + #[test] + fn t6040_auxiliary_tiler_storage_is_not_tail_pointer_cache() { + // CTS point_size_10000 renders to 10240 squared. Its GTP sub-buffer + // is required by the existing 25G83 program, not a larger cache. + for utile in [16, 32] { + let layout = Layout::new(&pass(10240, utile), 2, RegisterPrograms::V25G83).unwrap(); + assert!(layout.tpc_size < cfg::TPC_MAX); + assert!(layout.record_stride().unwrap() < cfg::TARGET_RECORD_MAX); + assert!(layout.tpc_end > cfg::TPC_MAX); + assert!(layout.tpc_capacity().unwrap() >= layout.tpc_end); + assert_eq!(layout.gtp[0], Some(0)); + assert!(layout.gtp[1].unwrap() >= layout.tpc_size); + } + } + + #[test] + fn core_and_tile_record_limits_still_reject_oversized_targets() { + let mut layout = Layout::new(&pass(10240, 32), 2, RegisterPrograms::V25G83).unwrap(); + layout.tpc_size = cfg::TPC_MAX + 1; + assert_eq!(layout.tpc_capacity(), Err(ERANGE)); + let mut layered = pass(16384, 16); + layered.layers = 2; + let layout = Layout::new(&layered, 2, RegisterPrograms::V25G83).unwrap(); + assert!(layout.record_stride().unwrap() > cfg::TARGET_RECORD_MAX); + assert_eq!(layout.tpc_capacity(), Err(ERANGE)); + } + + #[test] + fn neo_keeps_the_same_cache_extent_and_limit() { + let mut layout = Layout::new(&pass(10240, 32), 1, RegisterPrograms::G17P).unwrap(); + assert!(layout.gtp.iter().all(Option::is_none)); + assert_eq!(layout.tpc_capacity().unwrap(), + aligned(layout.tpc_size, mmu::UAT_PGSZ).unwrap().max(cfg::TPC_MIN)); + layout.tpc_size = cfg::TPC_MAX + 1; + layout.tpc_end = layout.tpc_size; + assert_eq!(layout.tpc_capacity(), Err(ERANGE)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index bcbbb2e7e875d..57406e5487a37 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -55,6 +55,11 @@ impl kernel::debugfs::Writer for View { let state = self.0.state.lock(); let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; writeln!(f, "online=1")?; + let (converted, published) = super::sync_input::counters(); + writeln!(f, "fw_inputs converted={} published={} native={} pending={}", + converted, published, super::sync_input::native_count(), + super::sync_input::pending_count())?; + writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?; firmware.queues.write_capacity(f)?; firmware.pool.write_capacity(f)?; writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 4f5b87bc853dc..90f6d77780bff 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -13,7 +13,12 @@ use super::{ recovery::Class, status::VmStatus, }; -use crate::{driver::AsahiDrmDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; +use crate::{ + driver::AsahiDrmDevice, + file, + hw::t8140::{dynamic, resources as cfg}, + mmu, +}; use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, @@ -27,9 +32,9 @@ mod publication; pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { - // Keep the existing GPU alias alive along with its prepared CPU view. - _mapping: Arc, - cpu: mmu::TimestampMapping, + // The buffer keeps its GPU alias and its CPU mapping alive. + buffer: Arc, + offset: usize, } impl Destination { @@ -45,13 +50,13 @@ impl Destination { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(mapping) = object; + let file::Object::TimestampBuffer(buffer) = object; let offset: usize = timestamp.offset.try_into()?; - if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { + if offset.checked_add(size_of::()).ok_or(EINVAL)? > buffer.size() { return Err(ERANGE); } - let cpu = mapping.timestamp_mapping(offset)?; - Ok(Some(Self { _mapping: mapping, cpu })) + let offset = buffer.timestamp_offset(offset)?; + Ok(Some(Self { buffer, offset })) } fn write(&self, ticks: u64) { @@ -61,13 +66,14 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.cpu.write(ns); + self.buffer.write_timestamp(self.offset, ns); barrier(Ordering::Release); } } -/// Timestamp mappings resolved and retained before any command is enqueued. -pub(crate) struct Destinations(Option; 4]>>); +/// Timestamp destinations resolved and retained before any command is enqueued. Dropping +/// one releases only a buffer reference; the buffer defers its own mapping release. +pub(crate) struct Destinations([Option; 4]); impl Destinations { pub(crate) fn resolve( @@ -82,20 +88,15 @@ impl Destinations { destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; } - // Empty submissions need no cleanup allocation. Timestamped commands - // prepare the unique work item before any output fence is published. - let retained = if destinations.iter().any(Option::is_some) { - Some(crate::cleanup::Deferred::new(destinations)?) - } else { - None - }; - Ok(Self(retained)) + Ok(Self(destinations)) + } + + pub(crate) fn has_outputs(&self) -> bool { + self.0.iter().any(Option::is_some) } fn iter(&self) -> core::slice::Iter<'_, Option> { - let destinations: &[Option] = - self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); - destinations.iter() + self.0.iter() } } @@ -254,6 +255,8 @@ pub(crate) struct Completion { terminal: Option>, compute: Option, feed: Option>, + usage: Arc, + sources: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -267,6 +270,9 @@ pub(crate) struct Completion { /// A recovery blamed this command. Unlike `class`, which a later /// classification can change, this never clears. blamed: AtomicBool, + /// A firmware restart listed this command as in flight. Until a replay + /// republishes it from the start, it never reports success. + interrupted: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } @@ -286,16 +292,21 @@ impl Completion { compute: bool, host_predecessors: bool, feed: Option>, + sources: Option>, + usage: Arc, ) -> Result> { let fence = fence::independent(contexts)?; - let ordered = !compute - && (host_predecessors || destinations.iter().any(Option::is_some)); + let ordered = !compute && (host_predecessors || destinations.iter().any(Option::is_some)); let terminal = if host_predecessors { Some(fence::independent(contexts)?) } else { None }; - let output = Output { fence, member, destinations }; + let output = Output { + fence, + member, + destinations, + }; let output = if ordered { HostOutput::Ordered(Publication::new(output, context.status().clone())?) } else { @@ -313,6 +324,8 @@ impl Completion { terminal, compute, feed, + usage, + sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -322,6 +335,7 @@ impl Completion { replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), blamed: AtomicBool::new(false), + interrupted: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -333,7 +347,9 @@ impl Completion { Fence::from_fence(&self.output.output().fence) } pub(crate) fn scheduler_fence(&self) -> Fence { - self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) + self.terminal + .as_ref() + .map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) } pub(crate) fn publication(&self) -> Option> { match &self.output { @@ -357,11 +373,51 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } + pub(super) fn sources(&self) -> Option<&Arc> { + self.sources.as_ref() + } + pub(super) fn is_input_source(&self, source: &Arc) -> bool { + self.output.output().member.is_source(source) + } + pub(super) fn set_source_ready(&self, ready: Fence) { + self.output.output().member.set_source_ready(ready); + } + pub(super) fn needs_source_ready(&self) -> bool { + self.output.output().member.needs_source_ready() + } + pub(super) fn note_input_error(&self, error: Error) -> bool { + self.output.output().member.note_error(error) + } + /// Called under the device mutex as a successful retirement is queued. + pub(super) fn note_source_retired(&self) { + if self.interrupted() { + // complete() fails it; consumers must not take its output as valid. + self.note_input_error(ECANCELED); + return; + } + match self.check_source_inputs() { + Ok(()) => self.output.output().member.note_retired(), + Err(error) => { + self.note_input_error(error); + } + } + } + pub(super) fn check_source_inputs(&self) -> Result { + if let Some(sources) = &self.sources { + sources.check()?; + } + Ok(()) + } + pub(crate) fn work_state(&self) -> Result> { self.output.output().member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { - self.output.output().destinations.iter().any(Option::is_some) + self.output + .output() + .destinations + .iter() + .any(Option::is_some) } pub(crate) fn compute_timestamp_va(&self) -> Result { @@ -397,6 +453,9 @@ impl Completion { .is_ok(); if accepted { self.replay_timeout.store(true, Ordering::Release); + // Callers replay only work proven not to have started; the new + // publication runs from the beginning and reports its own result. + self.interrupted.store(false, Ordering::Release); } accepted } @@ -485,6 +544,35 @@ impl Completion { self.blamed.load(Ordering::Acquire) } + /// A firmware restart listed this command in its recovery table: it was in + /// flight, so it may have run partly. Its stamps and completion record may + /// still look like success. Unless a replay republishes it from the start, + /// it fails and the failure is always reported to the client. + pub(crate) fn note_interrupted(&self) { + self.interrupted.store(true, Ordering::Release); + } + + pub(crate) fn interrupted(&self) -> bool { + self.interrupted.load(Ordering::Acquire) + } + + /// Whether this failure makes the VM unusable. Spared work, and interrupted + /// work no recovery found guilty, fail without poisoning it. + pub(crate) fn poisons(&self, error: Error) -> bool { + let interrupted = self.interrupted(); + (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) + } + + /// ECANCELED stays silent only for work that was never interrupted (closed + /// or cancelled by its owner); every other failure reaches the client. + fn report(&self, error: Error) { + if self.poisons(error) { + self.status.record(error); + } else if error != ECANCELED || self.interrupted() { + self.status.report_failure(error); + } + } + /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -515,6 +603,13 @@ impl Completion { .map(super::queue::admission::RenderLease::slot) } + /// Called under the device mutex once the render's kicks are committed. + pub(crate) fn note_render_publication(&self, fragment: super::fw::kick::KickDependency) { + if let Some(lease) = self.render.lock().as_ref() { + lease.note_publication(fragment); + } + } + /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { @@ -525,6 +620,9 @@ impl Completion { /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { + if let Some(sources) = &self.sources { + sources.release(); + } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -533,7 +631,15 @@ impl Completion { drop(lease); } + /// Whether the command still holds its VM pin: firmware may still own or replay it. + pub(crate) fn retains_vm(&self) -> bool { + self.vm_job.lock().is_some() + } + fn release_healthy(&self) { + if let Some(sources) = &self.sources { + sources.release(); + } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -541,22 +647,40 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { + if result.is_ok() { + if let Err(error) = self.check_source_inputs() { + result = Err(error); + } + } if self.signalled.swap(true, Ordering::AcqRel) { return; } + if result.is_ok() && self.interrupted() { + result = Err(ECANCELED); + } + if let Ok(ticks) = &result { + use crate::usage::Engine; + let hz = cfg::COMMAND_TIMESTAMP_HZ as u32; + if self.compute.is_some() { + self.usage.add(Engine::Compute, ticks[0], ticks[1], hz); + } else { + self.usage.add(Engine::Vertex, ticks[0], ticks[1], hz); + self.usage.add(Engine::Fragment, ticks[2], ticks[3], hz); + } + } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { - feed.note_render_pass(ticks[2], ticks[3]); + // Compute reports its dispatch envelope in the first pair, a + // render its fragment pass in the second. + if self.compute.is_some() { + feed.note_pass(ticks[0], ticks[1]); + } else { + feed.note_pass(ticks[2], ticks[3]); + } } if let Err(error) = result { // Spared work cancelled after another VM's recovery is not this client's device loss. - if self.spared() { - if error != ECANCELED { - self.status.report_failure(error); - } - } else if error != ECANCELED { - self.status.record(error); - } + self.report(error); } self.release_healthy(); self.signal_terminal(result); @@ -570,7 +694,9 @@ impl Completion { return; } self.status.record_if_device_loss(error); - if self.accepted.load(Ordering::Acquire) { + // As in complete(): a cancellation is not this client's device loss. A guilty VM + // already carries its own sticky error. + if self.accepted.load(Ordering::Acquire) && error != ECANCELED { self.status.report_failure(error); } self.release_healthy(); @@ -589,9 +715,11 @@ impl Completion { guard.quarantine(); } if self.spared() { - self.status.report_failure(error); - } else if error != ECANCELED { - self.status.record(error); + if error != ECANCELED || self.interrupted() { + self.status.report_failure(error); + } + } else { + self.report(error); } self.signal_terminal(Err(error)); self.output.finish(Err(error)); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index c93ab25d55ce3..8a6d9b572dec0 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -14,8 +14,9 @@ use super::{ status::VmStatus, }; use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; -use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}; +use identity::{Order, RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; +use super::job::Engine; use kernel::{ new_mutex, prelude::*, @@ -81,6 +82,19 @@ impl WorkStorage { fence(Ordering::Release); Ok(()) } + /// Whether the firmware completed every kick counted so far on node `word`. + fn drained(&self, word: u32) -> Result { + let slot = (word & 63) as usize; + if self.words.get(slot) != Some(&Some(word)) { + return Err(EINVAL); + } + let completed = self + .object + .word(slot * NODE_STRIDE + core::mem::offset_of!(fw::WorkNode, completed))? + .load(Ordering::Relaxed); + fence(Ordering::Acquire); + Ok(completed >= self.counts[slot]) + } fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { let slot = (word & 63) as usize; if self.words.get(slot) != Some(&Some(word)) { @@ -114,9 +128,24 @@ pub(crate) struct Context { scheduler: Arc, scheduler_generation: u32, published: AtomicBool, + /// A kill of this context was published, through a compute queue or on its render pairs' + /// behalf. A context is killed at most once. + kill_published: AtomicBool, + /// Advisory only: publication still checks the physical binding under the + /// device mutex. False skips an inline attempt known to need admission. + compute_binding_hint: AtomicBool, + /// Mixed-engine contexts keep the original inline publication path. + render_seen: AtomicBool, qos: QosLease, owner_pid: u32, states: WorkStates, + /// Signalled when a work state returns; publishers that found none wait on it. + state_returns: Arc, + /// The domain refused compute publishers wait on; signalled with `state_returns` when the + /// admission turn moves on. + compute_returns: Arc, + /// The order in which this context's ioctls take work-state nodes. + order: Arc>, work: Arc>, work_base: u64, render: Arc>>>, @@ -128,6 +157,10 @@ pub(crate) struct Context { policy: Policy, } impl Context { + #[expect( + clippy::too_many_arguments, + reason = "A context takes its allocators, identities and both admission domains." + )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -135,6 +168,8 @@ impl Context { execution: Arc, owner_pid: u32, policy: Policy, + state_returns: Arc, + compute_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -183,15 +218,23 @@ impl Context { GFP_KERNEL, )?; let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?; + let order = + Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { execution, status, scheduler, scheduler_generation, published: AtomicBool::new(false), + kill_published: AtomicBool::new(false), + compute_binding_hint: AtomicBool::new(false), + render_seen: AtomicBool::new(false), qos, owner_pid, states: WorkStates::new(), + state_returns, + compute_returns, + order, work, work_base, render, @@ -203,6 +246,22 @@ impl Context { pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_binding_hint(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + || self.compute_binding_hint.load(Ordering::Relaxed) + } + /// This logical queue has submitted render work, so its compute may have a render prefix. + pub(crate) fn render_seen(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + } + pub(crate) fn note_render(&self) { + if !self.render_seen.load(Ordering::Relaxed) { + self.render_seen.store(true, Ordering::Release); + } + } + pub(crate) fn note_compute_binding(&self, owned: bool) { + self.compute_binding_hint.store(owned, Ordering::Relaxed); + } pub(crate) fn compute_timestamps(&self) -> &Arc { &self.compute_timestamps } @@ -253,6 +312,28 @@ impl Context { _work: self.work.clone(), } } + /// Called once the firmware consumed this context's scheduler release. If its work list + /// still links a node, the firmware keeps referencing that node: on Neo a render command + /// that ran behind a failed compute command is never unlinked, and a later context given + /// the same work storage (the object cache hands it out again) made the firmware recover + /// the GPU. Keep the storage mapped and out of reuse until firmware stop; returns whether + /// it was linked. + pub(crate) fn retain_linked_work(&self, pool: &Arc) -> bool { + // An unreadable head counts as linked: retaining is the safe side. + let linked = self.work_head().map_or(true, |head| head != [0, self.work_head_va()]); + if linked { + self.work.lock().object.retain_until_stop(pool); + } + linked + } + + /// The firmware may still walk this context's work list into memory it names: a node is + /// linked and the scheduler page shows no kill. Unreadable state counts as linked. + pub(crate) fn firmware_may_link(&self) -> bool { + let killed = super::queue::compute::exit::State::read(self).map_or(0, |state| state.killed); + must_retain(self.work_head().ok(), self.work_head_va(), killed) + } + pub(crate) fn work_head_va(&self) -> u64 { self.work_base + (UAT_PGSZ + 0x400) as u64 } @@ -267,6 +348,16 @@ impl Context { Ok(head) } + /// Record that a kill of this context was published (see `kill_published`). + pub(crate) fn note_kill_published(&self) { + self.kill_published.store(true, Ordering::Release); + } + + /// A kill of this context was published. + pub(crate) fn kill_published(&self) -> bool { + self.kill_published.load(Ordering::Acquire) + } + pub(crate) fn is_published(&self) -> bool { self.published.load(Ordering::Acquire) } @@ -333,41 +424,60 @@ impl Context { pub(crate) struct WorkStateLease { context: Arc, word: AtomicU32, + /// Index of the engine whose admission took the node, valid once `word` is set. + engine: AtomicU8, + /// The ioctl's place in its context's submission order. + ordinal: u64, } impl WorkStateLease { + pub(super) fn status(&self) -> &Arc { self.context.status() } pub(crate) fn deferred(context: Arc) -> Self { + let ordinal = context.order.lock().assign(); Self { context, word: AtomicU32::new(FREE), + engine: AtomicU8::new(0), + ordinal, } } - pub(crate) fn acquire(&self) -> Result { + /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: too many + /// older ioctls still wait for theirs (see `identity::Order`), or no node may be taken now + /// (see `identity::may_take`); an older ioctl's admission or a return lets it try again. + pub(crate) fn acquire(&self, engine: Engine) -> Result { let current = self.word(); if current != FREE { return Ok(current); } - let Some(candidate) = self.context.states.allocate() else { - let current = self.word(); - return if current != FREE { - Ok(current) - } else { - Err(EBUSY) - }; - }; - if let Err(error) = self.context.work.lock().initialize(candidate) { - self.context.states.release(candidate); + // The work-storage lock serializes the decision with this context's other + // admissions, and with the other engine of this ioctl taking its node. + let mut work = self.context.work.lock(); + let current = self.word(); + if current != FREE { + return Ok(current); + } + let mut order = self.context.order.lock(); + if !order.admits(self.ordinal) { + return Err(EBUSY); + } + let engine = engine.index(); + let candidate = self.context.states.allocate(engine).ok_or(EBUSY)?; + if let Err(error) = work.initialize(candidate) { + self.context.release_state(candidate, engine); return Err(error); } - match self - .word - .compare_exchange(FREE, candidate, Ordering::AcqRel, Ordering::Acquire) - { - Ok(_) => Ok(candidate), - Err(installed) => { - self.context.states.release(candidate); - Ok(installed) - } + self.engine.store(engine as u8, Ordering::Relaxed); + self.word.store(candidate, Ordering::Release); + let wake = order.admit(self.ordinal); + drop(order); + drop(work); + if wake { + self.context.pass_turn(); } + Ok(candidate) + } + /// The domain a publisher samples before acquire() and waits on after EBUSY. + pub(crate) fn returns(&self) -> &Arc { + &self.context.state_returns } pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) @@ -379,6 +489,16 @@ impl WorkStateLease { } Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) } + /// Whether the firmware completed every kick counted so far on this ioctl's node. It stores + /// the completion stamps of the node's commands only then, so a command whose record is + /// present still waits for its stamp while other commands of its ioctl run. + pub(crate) fn drained(&self) -> Result { + let word = self.word(); + if word == FREE { + return Err(EINVAL); + } + self.context.work.lock().drained(word) + } pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { self.context .work @@ -409,21 +529,55 @@ impl Drop for WorkStateLease { fn drop(&mut self) { let word = self.word(); if word != FREE { - self.context.states.release(word); + self.context.release_state(word, usize::from(self.engine.load(Ordering::Relaxed))); + } else if self.context.order.lock().leave(self.ordinal) { + self.context.pass_turn(); } } } +impl Context { + /// Every return bumps the domain: a publisher that sampled it before its + /// failed acquire() then never sleeps through this node. + fn release_state(&self, word: u32, engine: usize) { + if self.states.release(word, engine) { + self.state_returns.notify(); + } + } + /// A refused publisher waits on its engine's domain. Bumping both once the turn moved past a + /// refusal means one that sampled before its refused acquire() never sleeps through it. + fn pass_turn(&self) { + self.state_returns.notify(); + self.compute_returns.notify(); + } +} + impl QosIds { pub(crate) fn used(&self) -> u32 { (*self.used.lock() & !3u128).count_ones() } } +/// Memory a context's work list may lead the firmware to stays allocated: a linked node (or +/// an unreadable head) and no acknowledged kill. +fn must_retain(head: Option<[u64; 2]>, head_va: u64, killed: u32) -> bool { + killed == 0 && head.is_none_or(|head| head != [0, head_va]) +} + #[kunit_tests(asahi_g17_qos_capacity)] mod capacity_tests { use super::*; + #[test] + fn linked_unkilled_work_is_retained() { + let va = 0xfffffc20_0000_4400; + assert!(!must_retain(Some([0, va]), va, 0)); + assert!(must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 0)); + assert!(must_retain(None, va, 0)); + assert!(!must_retain(Some([0xfffffc20_0000_0000, 0xfffffc20_0000_0098]), va, 1)); + assert!(!must_retain(None, va, 1)); + } + #[test] fn failed_client_construction_returns_every_qos_id() { let ids = QosIds::new().unwrap(); diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 595c7154d42ed..7586f8474e602 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -4,23 +4,50 @@ //! the complete sequence distinguishes its owner until exact retirement. use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::prelude::*; pub(super) const FREE: u32 = u32::MAX; pub(super) const STATE_COUNT: usize = 63; pub(super) const NODE_STRIDE: usize = 0x100; +/// Engines whose admissions take work-state nodes: render (0) and compute (1). +pub(super) const ENGINES: usize = 2; + +/// Whether an engine that holds `held` nodes may take one of `free` nodes. +/// +/// All engines of one ioctl share its node, and the node returns only once every command of +/// the ioctl has retired. So render admission can keep taking nodes for newer ioctls while +/// their compute commands still wait in the compute queue behind an older ioctl that has no +/// node yet; once render held every node, nothing could retire and both queues stalled. An +/// engine therefore never takes the last free node while it already holds one. The other +/// engine can then always take a node, and an engine refused here waits only for nodes it +/// took itself, for ioctls ahead of its waiting command. Each engine admits a context's +/// ioctls in submission order, so those waits cannot form a cycle. +pub(super) const fn may_take(free: usize, held: u32) -> bool { + free > 1 || (free == 1 && held == 0) +} + pub(super) struct WorkStates { next: AtomicU32, live: [AtomicU32; 64], + /// Nodes taken by each engine's admissions and not yet released. + held: [AtomicU32; ENGINES], } impl WorkStates { pub(super) const fn new() -> Self { Self { next: AtomicU32::new(0), live: [const { AtomicU32::new(FREE) }; 64], + held: [const { AtomicU32::new(0) }; ENGINES], } } - pub(super) fn allocate(&self) -> Option { + /// Takes a node for `engine`, or none when [`may_take`] refuses. Callers serialize + /// allocations of one context; releases may run concurrently. + pub(super) fn allocate(&self, engine: usize) -> Option { + let held = self.held.get(engine)?; + if !may_take(STATE_COUNT.saturating_sub(self.used()), held.load(Ordering::Acquire)) { + return None; + } for _ in 0..64 { let word = self.next.fetch_add(1, Ordering::Relaxed); let index = (word & 63) as usize; @@ -29,17 +56,119 @@ impl WorkStates { .compare_exchange(FREE, word, Ordering::AcqRel, Ordering::Acquire) .is_ok() { + held.fetch_add(1, Ordering::AcqRel); return Some(word); } } None } - pub(super) fn release(&self, word: u32) -> bool { + pub(super) fn release(&self, word: u32, engine: usize) -> bool { let index = (word & 63) as usize; - index != STATE_COUNT + let released = index != STATE_COUNT && self.live[index] .compare_exchange(word, FREE, Ordering::AcqRel, Ordering::Acquire) - .is_ok() + .is_ok(); + if released { + if let Some(held) = self.held.get(engine) { + held.fetch_sub(1, Ordering::AcqRel); + } + } + released + } +} + +/// Ordinals past the turn remembered for ioctls that took a node or left early. +pub(super) const ORDER_WINDOW: u64 = 1024; + +/// How far past the oldest waiting ioctl an ioctl may take its node. +pub(super) const RUN_AHEAD: u64 = 8; + +/// The order in which a context's ioctls take work-state nodes. +/// +/// Each engine admits a context's ioctls in submission order, but the engines advance +/// independently: render admission can take nodes for ioctls far ahead of an older ioctl still +/// waiting for its turn on the compute queue. Under deep mixed load (128 compute and 64 render +/// ioctls in flight on one queue) the firmware then stops completing all of the context's work +/// within seconds. Native macOS schedules a queue's command buffers in commit order across +/// engines, at most 36 between scheduling and completion, and the same load runs there without +/// failure. +/// +/// So an ioctl takes its node only while at most [`RUN_AHEAD`] older ioctls of its context have +/// neither taken one nor left without one. Native scheduling happens at commit, on one thread; +/// here each engine admits from its own scheduler, and a strictly ordered turn costs a scheduler +/// round trip whenever the other engine's ioctl is ready first. A short run-ahead avoids that +/// without letting one engine get far ahead of the other. +pub(super) struct Order { + /// The next ordinal to hand out. + next: u64, + /// The oldest ordinal that has neither taken a node nor left. + turn: u64, + /// Ordinals past the turn that took a node or left, as bits at ordinal % ORDER_WINDOW. + done: [u64; (ORDER_WINDOW / 64) as usize], + /// Set once an ioctl left too far past the turn to remember; admission is unordered after. + unordered: bool, + /// An ioctl was refused since the turn last moved: its publisher waits for the next move. + refused: bool, +} +impl Order { + pub(super) const fn new() -> Self { + Self { + next: 0, + turn: 0, + done: [0; (ORDER_WINDOW / 64) as usize], + unordered: false, + refused: false, + } + } + /// The ordinal of a new ioctl, in submission order. + pub(super) fn assign(&mut self) -> u64 { + let ordinal = self.next; + self.next += 1; + ordinal + } + /// Whether the ioctl `ordinal` may take its node now. A refusal is remembered until the + /// turn next moves, so that move wakes the refused publisher. + pub(super) fn admits(&mut self, ordinal: u64) -> bool { + let admits = self.unordered || ordinal <= self.turn + RUN_AHEAD; + self.refused |= !admits; + admits + } + /// The ioctl `ordinal` took its node. Returns whether a refused ioctl may now be admitted. + pub(super) fn admit(&mut self, ordinal: u64) -> bool { + self.finish(ordinal) + } + /// The ioctl `ordinal` left without taking a node (it failed or was cancelled first). + /// Returns whether a refused ioctl may now be admitted. + pub(super) fn leave(&mut self, ordinal: u64) -> bool { + self.finish(ordinal) + } + fn finish(&mut self, ordinal: u64) -> bool { + if self.unordered || ordinal < self.turn { + return false; + } + if ordinal == self.turn { + self.pass(); + return core::mem::take(&mut self.refused); + } + if ordinal - self.turn >= ORDER_WINDOW { + self.unordered = true; + return core::mem::take(&mut self.refused); + } + let bit = ordinal % ORDER_WINDOW; + self.done[(bit / 64) as usize] |= 1 << (bit % 64); + false + } + /// Moves the turn past the current ordinal and every following one already done. + fn pass(&mut self) { + loop { + self.turn += 1; + let bit = self.turn % ORDER_WINDOW; + let (word, mask) = ((bit / 64) as usize, 1u64 << (bit % 64)); + if self.done[word] & mask == 0 { + break; + } + self.done[word] &= !mask; + } } } @@ -78,3 +207,122 @@ impl WorkStates { impl RenderSlots { pub(super) fn used(&self) -> usize { self.pairs.iter().flatten().count() } } + +#[kunit_tests(asahi_g17_work_states)] +mod tests { + use super::*; + + #[test] + fn last_node_stays_with_an_engine_holding_none() { + assert!(!may_take(0, 0)); + assert!(may_take(1, 0)); + assert!(!may_take(1, 3)); + assert!(may_take(2, 3)); + } + + #[test] + fn render_cannot_starve_compute_of_a_node() { + let states = WorkStates::new(); + let mut render = KVec::new(); + while let Some(word) = states.allocate(0) { + render.push(word, GFP_KERNEL).unwrap(); + } + // Render stops one short; compute takes the last node, then nothing is left. + assert_eq!(render.len(), STATE_COUNT - 1); + let compute = states.allocate(1).unwrap(); + assert_eq!(states.used(), STATE_COUNT); + assert!(states.allocate(0).is_none()); + assert!(states.allocate(1).is_none()); + // A lone free node goes to the engine holding none. + assert!(states.release(compute, 1)); + assert!(states.allocate(0).is_none()); + let compute = states.allocate(1).unwrap(); + // With both engines holding nodes, a lone free node waits for one of their returns. + assert!(states.release(render.pop().unwrap(), 0)); + assert!(states.allocate(0).is_none()); + assert!(states.allocate(1).is_none()); + // Two free nodes: render may take one again, never the last. + assert!(states.release(render.pop().unwrap(), 0)); + assert!(states.allocate(0).is_some()); + assert!(states.allocate(0).is_none()); + assert!(states.release(compute, 1)); + assert!(states.allocate(1).is_some()); + } + + #[test] + fn ioctls_take_nodes_at_most_run_ahead_past_the_oldest_waiting_one() { + let mut order = Order::new(); + let ordinals: [u64; RUN_AHEAD as usize + 3] = core::array::from_fn(|_| order.assign()); + let (oldest, last) = (ordinals[0], ordinals[RUN_AHEAD as usize + 2]); + // Every ioctl up to RUN_AHEAD past the oldest waiting one may take its node. + for &ordinal in &ordinals[..=RUN_AHEAD as usize] { + assert!(order.admits(ordinal)); + } + assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); + // Ioctls ahead of the turn taking their nodes do not move it. + for &ordinal in &ordinals[1..=RUN_AHEAD as usize] { + assert!(!order.admit(ordinal)); + } + assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); + // The oldest one moves the turn past everything already admitted, and wakes the refused. + assert!(order.admit(oldest)); + assert!(order.admits(last)); + let next = order.assign(); + assert!(order.admits(next)); + assert!(!order.admits(next + RUN_AHEAD + 1)); + } + + #[test] + fn ioctls_leaving_without_a_node_pass_the_turn() { + let mut order = Order::new(); + let (a, b, c) = (order.assign(), order.assign(), order.assign()); + let later: [u64; RUN_AHEAD as usize] = core::array::from_fn(|_| order.assign()); + let beyond = later[RUN_AHEAD as usize - 1]; + // b and c fail before their turn; while a waits, ioctls past a + RUN_AHEAD wait too. + assert!(!order.leave(b)); + assert!(!order.leave(c)); + assert!(!order.admits(beyond)); + // Once a takes its node, the turn passes b and c as well, and the refused one may retry. + assert!(order.admit(a)); + assert!(order.admits(beyond)); + // With nobody refused since, moving the turn wakes nobody; an ordinal already passed + // leaves without effect. + assert!(!order.leave(later[0])); + assert!(!order.leave(a)); + } + + #[test] + fn an_ioctl_leaving_too_far_ahead_unorders_admission() { + let mut order = Order::new(); + let first = order.assign(); + let mut last = first; + for _ in 0..ORDER_WINDOW { + last = order.assign(); + } + assert!(!order.admits(last)); + assert!(order.leave(last)); + assert!(order.admits(last - 1)); + } + + #[test] + fn single_engine_admission_wakes_nobody() { + let mut order = Order::new(); + for _ in 0..3 * RUN_AHEAD { + let ordinal = order.assign(); + assert!(order.admits(ordinal)); + assert!(!order.admit(ordinal)); + } + } + + #[test] + fn releases_return_nodes_to_their_engine() { + let states = WorkStates::new(); + let word = states.allocate(1).unwrap(); + assert!(states.release(word, 1)); + assert!(!states.release(word, 1)); + assert_eq!(states.used(), 0); + assert_eq!(states.held[1].load(Ordering::Acquire), 0); + // An engine outside the table takes nothing. + assert!(states.allocate(ENGINES).is_none()); + } +} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs index 0bc1d5024566e..5a0ed611b1dde 100644 --- a/drivers/gpu/drm/asahi/g17/dependency.rs +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -9,6 +9,7 @@ use super::{ use kernel::prelude::*; /// A failed producer does not establish a fault in its consumer's VM. +#[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) fn inherited_error(error: Error) -> Error { if error == EIO || error == ENODEV || error == ETIMEDOUT { ECANCELED @@ -17,14 +18,61 @@ pub(crate) fn inherited_error(error: Error) -> Error { } } +/// What a G17 consumer of a failed producer fails with, whatever the producer failed with: its +/// client does not see a cancellation as a lost device. A producer a recovery dropped fails with +/// ENODATA, and passing that on made every consumer of the producer's buffers lose its device too. +pub(crate) const PRODUCER_FAILED: Error = ECANCELED; + /// A replay's original host inputs must still be signalled. Treat an unmet /// input as a failed replay, never as a wait under the device mutex. pub(crate) fn replay_status(status: i32) -> Result { if status == 0 { Err(EBUSY) } - else if status < 0 { Err(inherited_error(Error::from_errno(status))) } + else if status < 0 { Err(PRODUCER_FAILED) } else { Ok(()) } } +/// Whether a failed command may still run or write: it holds its VM pin until it has physically +/// retired, or was never published. +pub(crate) trait Retention { + fn retained(&self) -> bool; +} + +/// The failed ordinals of one engine that later commands must not bypass. A command whose prefix +/// covers one is cancelled, so none runs before a failed producer the firmware may still replay. +/// Once that producer has retired nothing of it can run or write any more, so it stops counting: +/// otherwise one cancelled command cancelled every later command of its queue for good. +pub(crate) struct FailureRecord { + retained: KVec<(u64, T)>, + /// A failure that could not be tracked: it keeps cancelling for good, as all failures once did. + untracked: Option<(u64, T)>, +} + +impl FailureRecord { + pub(crate) const fn new() -> Self { + Self { retained: KVec::new(), untracked: None } + } + + /// A finished frontier entry, at pruning. + pub(crate) fn note(&mut self, sequence: u64, status: i32, entry: &T) { + if status >= 0 || !entry.retained() { + return; + } + if self.retained.push((sequence, entry.clone()), GFP_KERNEL).is_err() + && self.untracked.as_ref().is_none_or(|(first, _)| sequence < *first) + { + self.untracked = Some((sequence, entry.clone())); + } + } + + /// The earliest failure later prefixes must not bypass. + pub(crate) fn earliest(&mut self) -> Option<(u64, &T)> { + self.retained.retain(|(_, entry)| entry.retained()); + Iterator::chain(self.retained.iter(), self.untracked.iter()) + .min_by_key(|(sequence, _)| *sequence) + .map(|(sequence, entry)| (*sequence, entry)) + } +} + pub(crate) struct Frontier { dependencies: [KickDependency; KICK_BARRIERS_MAX], sequences: [u64; KICK_BARRIERS_MAX], @@ -89,3 +137,79 @@ impl Frontier { self.insert(sequence, dependency) } } + +#[kunit_tests(asahi_g17_dependency)] +mod tests { + use super::*; + + #[derive(Clone)] + struct Probe(kernel::sync::Arc); + + impl Probe { + fn new(retained: bool) -> Self { + Self(kernel::sync::Arc::new(core::sync::atomic::AtomicBool::new(retained), GFP_KERNEL).unwrap()) + } + fn retire(&self) { + self.0.store(false, core::sync::atomic::Ordering::Relaxed); + } + } + + impl Retention for Probe { + fn retained(&self) -> bool { + self.0.load(core::sync::atomic::Ordering::Relaxed) + } + } + + fn earliest(record: &mut FailureRecord) -> Option { + record.earliest().map(|(sequence, _)| sequence) + } + + #[test] + fn a_cancelled_command_does_not_poison_its_queue() { + // Cancelled or failed before publication: it released its pin and can never run. + let mut record = FailureRecord::new(); + record.note(5, ECANCELED.to_errno(), &Probe::new(false)); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn a_retained_failure_cancels_until_it_retires() { + let mut record = FailureRecord::new(); + let failed = Probe::new(true); + record.note(5, EIO.to_errno(), &failed); + assert_eq!(earliest(&mut record), Some(5)); + failed.retire(); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn a_later_retained_failure_survives_an_earlier_retirement() { + let mut record = FailureRecord::new(); + let (first, second) = (Probe::new(true), Probe::new(true)); + record.note(5, EIO.to_errno(), &first); + record.note(9, ECANCELED.to_errno(), &second); + assert_eq!(earliest(&mut record), Some(5)); + first.retire(); + assert_eq!(earliest(&mut record), Some(9)); + second.retire(); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn successes_are_not_recorded() { + let mut record = FailureRecord::new(); + record.note(5, 1, &Probe::new(true)); + assert_eq!(earliest(&mut record), None); + } + + #[test] + fn failed_predecessor_cannot_be_bypassed() { + let mut frontier = Frontier::new(); + for error in [ECANCELED, EIO, ETIMEDOUT] { + assert_eq!(frontier.include(Some(5), 5, error.to_errno(), 0), Err(ECANCELED)); + } + assert_eq!(frontier.include(Some(5), 5, 1, 0), Ok(())); + assert_eq!(frontier.include(Some(5), 6, EIO.to_errno(), 0), Ok(())); + assert_eq!(frontier.count, 0); + } +} diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index 9508a827be4b1..a175e58392379 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -36,6 +36,8 @@ pub(super) trait Queues { fn retain_render_completions(&mut self, masks: [u64; 2]); fn complete_compute(&mut self, masks: [u64; 2]) -> Result; fn context_killed(&mut self, cookie: u64); + /// The firmware failed a compute command (see [`Event::ComputeFault`]). + fn compute_fault(&mut self, qid: u8, stamp: u32); } /// Work requested after releasing the device mutex. @@ -72,6 +74,12 @@ fn batch(host: &mut H, role: Role, masks: &mut [u64; 2]) -> Result { } Event::FreeListGrow(completion) => host.complete_freelist_grow(completion)?, Event::ContextKilled(cookie) => host.context_killed(cookie), + Event::ComputeFault { qid, stamp } => { + crate::g17::trace::teardown(1, u64::from(qid), u64::from(stamp), 0); + host.compute_fault(qid, stamp); + // Its queue retires or settles its quarantine on this pass. + masks[usize::from(qid) / 64] |= 1 << (qid % 64); + } Event::Unknown(_) => {} } Ok(true) diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 0045dff413922..bd1228a82e26f 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -44,6 +44,8 @@ struct Detector { busy: AtomicU64, streak: AtomicU32, until: AtomicU64, + /// Until when the GPU counts as the bottleneck (see `Feed::gpu_bound`). + bound_until: AtomicU64, active: AtomicBool, pending: AtomicBool, stopping: AtomicBool, @@ -57,6 +59,7 @@ impl Detector { busy: AtomicU64::new(0), streak: AtomicU32::new(0), until: AtomicU64::new(0), + bound_until: AtomicU64::new(0), active: AtomicBool::new(false), pending: AtomicBool::new(false), stopping: AtomicBool::new(false), @@ -108,17 +111,26 @@ impl Detector { Err(_) => 0, } }; - if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { + if streak < STREAK_ON { + return; + } + if idle_sum.saturating_mul(4) >= busy_sum { self.until.store(now + LINGER_NS, Ordering::Release); if !self.active.swap(true, Ordering::AcqRel) { self.pending.store(true, Ordering::Release); } + } else { + self.bound_until.store(now + LINGER_NS, Ordering::Release); } } fn wanted(&self, now: u64) -> bool { !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire) } + + fn bound(&self, now: u64) -> bool { + !self.stopping.load(Ordering::Acquire) && now < self.bound_until.load(Ordering::Acquire) + } } #[cfg(CONFIG_CPU_FREQ)] @@ -410,12 +422,23 @@ impl Feed { Monotonic::ktime_get() as u64 } - pub(crate) fn note_render_pass(&self, start: u64, end: u64) { + pub(crate) fn note_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); } } + /// Whether the GPU itself is the bottleneck: passes follow each other with refill gaps + /// under a quarter of their busy time, and the feed does not see host-refill-bound work. + /// Lingers like streaming, so a pause of more than a few milliseconds ends it. + pub(crate) fn gpu_bound(&self) -> bool { + if !cfg!(CONFIG_CPU_FREQ) { + return false; + } + let now = Self::now(); + self.detector.bound(now) && !self.detector.wanted(now) + } + /// Called after a firmware event pass, before queuing the single feed work item. pub(crate) fn take_pending(&self) -> bool { self.detector.pending.swap(false, Ordering::AcqRel) @@ -498,3 +521,51 @@ impl PinnedDrop for Feed { self.shutdown(); } } + +#[kernel::macros::kunit_tests(asahi_g17_feed_detector)] +mod detector_tests { + use super::*; + + /// Notes `passes` passes of `busy` ticks, each `gap` ticks after the previous, 1 ms apart. + fn run(detector: &Detector, passes: u64, busy: u64, gap: u64) -> u64 { + let mut start = 1000; + let mut now = 1_000_000; + for _ in 0..passes { + detector.note(start, start + busy, now); + start += busy + gap; + now += 1_000_000; + } + now + } + + #[test] + fn back_to_back_long_passes_are_gpu_bound() { + let detector = Detector::new(); + // 2 ms passes with 40 us refill gaps, as a deep heavy queue or a heavy row at depth 1. + let now = run(&detector, 16, CLOCK_HZ / 500, CLOCK_HZ / 25_000); + assert!(detector.bound(now)); + assert!(!detector.wanted(now)); + assert!(!detector.bound(now + LINGER_NS)); + } + + #[test] + fn refill_gaps_stream_instead() { + let detector = Detector::new(); + // 20 us passes with 60 us refill gaps: bound by host refills. + let now = run(&detector, 16, CLOCK_HZ / 50_000, CLOCK_HZ / 16_667); + assert!(detector.wanted(now)); + assert!(!detector.bound(now)); + } + + #[test] + fn paced_or_short_runs_are_neither() { + let detector = Detector::new(); + // Each pass starts after a pause of at least the refill limit. + let now = run(&detector, 16, CLOCK_HZ / 500, MAX_REFILL_TICKS); + assert!(!detector.bound(now) && !detector.wanted(now)); + // Fewer back-to-back passes than the streak needs. + let detector = Detector::new(); + let now = run(&detector, u64::from(STREAK_ON) - 1, CLOCK_HZ / 500, 0); + assert!(!detector.bound(now) && !detector.wanted(now)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 3b31c3cead064..9015dc82e4076 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -670,10 +670,18 @@ pub(crate) struct Owner { enum LeasePhase { Preparing, Ready, + /// No reference since the given time. The registration stays live so that + /// the owner's next render reuses it without a control-ring release; the + /// release becomes pending when the linger ends. + Lingering(u64), ReleaseSent(u32), Released, } +/// How long an idle registration of an open owner may stay live: the +/// firmware's effort idle hold. +pub(crate) const LINGER_NS: u64 = 16_000_000; + /// References to one prepopulated owner: preparation and two engine commands /// per render. Published releases do not gate reacquisition; their consumption /// witnesses are carried into the successor. @@ -707,10 +715,14 @@ impl Lease { } let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; let same_owner = previous.owner == owner; + // A lingering registration was never released: it continues like an + // overlap of the owner's previous render. let overlapping = same_owner - && previous.phase == LeasePhase::Ready - && previous.references >= 2 - && previous.commands >= 2; + && match previous.phase { + LeasePhase::Ready => previous.references >= 2 && previous.commands >= 2, + LeasePhase::Lingering(_) => true, + _ => false, + }; let reacquire = previous.reacquirable(); if !overlapping && !reacquire { return Err(EBUSY); @@ -772,16 +784,35 @@ impl Lease { Err(EIO) } /// Both engines have retired this render's exact retained submission. + /// With `linger`, a registration left without references stays live + /// instead of making its final release pending. pub(crate) fn complete_render( &mut self, + linger: bool, ) -> Result> { if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { return Err(EINVAL); } self.references -= 2; self.commands -= 2; + if self.references == 0 && linger { + self.phase = LeasePhase::Lingering(super::now_ns().max(1)); + } Ok(self.pending_release()) } + /// The time since which an idle registration lingers, if it does. + pub(crate) fn lingering(&self) -> Option { + match self.phase { + LeasePhase::Lingering(since) => Some(since), + _ => None, + } + } + /// Ends a linger; the final release becomes pending. + pub(crate) fn end_linger(&mut self) { + if matches!(self.phase, LeasePhase::Lingering(_)) { + self.phase = LeasePhase::Ready; + } + } pub(crate) fn pending_release(&self) -> Option { (self.phase == LeasePhase::Ready && self.references == 0).then(|| { super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index 524132c4ae7be..26c6adcf34c83 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -136,6 +136,9 @@ pub(crate) enum Event { TvbGrow(TvbGrowRequest), FreeListGrow(FreeListCompletion), ContextKilled(u64), + /// The firmware failed a compute command of queue `qid` and stores no completion stamp for + /// it: `stamp` is the value that command would have stored. + ComputeFault { qid: u8, stamp: u32 }, Unknown(u32), } @@ -145,6 +148,8 @@ impl EventRecord { const TVB_GROW: u32 = 6; const FREE_LIST_GROW: u32 = 13; const CONTEXT_KILLED: u32 = 12; + const COMPUTE_FAULT: u32 = 7; + const COMPUTE_QIDS: u32 = 128; const DIAGNOSTIC_PAIRS_MAX: u16 = 5; const TVB_SLOTS: u32 = 127; const SUBPIPES: u32 = 64; @@ -160,6 +165,16 @@ impl EventRecord { unsafe { self.words[0] } } + /// Type 7 is a retirement witness only on the firmware whose layout and + /// visibility semantics are qualified. 25G83 must keep consuming it as an + /// unknown event until its own fault contract is established. + pub(crate) fn decode_for(&self, graph: super::variant::InitGraph) -> Result { + if graph == super::variant::InitGraph::V25G83 && self.tag() == Self::COMPUTE_FAULT { + return Ok(Event::Unknown(Self::COMPUTE_FAULT)); + } + self.decode() + } + /// Validates the record's fields without interpreting resource ownership. /// A malformed known event is an error, unlike an unknown tag. pub(crate) fn decode(&self) -> Result { @@ -198,6 +213,19 @@ impl EventRecord { halt_count: record.halt_count, })) } + Self::COMPUTE_FAULT => { + // SAFETY: Every constructor initializes all integer words. + let words = unsafe { self.words }; + // Only compute faults are known to use this type: any other layout stays an + // unknown event, consumed and ignored as before. + if words[3] >= Self::COMPUTE_QIDS || words[4] & 0xff != 0 { + return Ok(Event::Unknown(Self::COMPUTE_FAULT)); + } + Ok(Event::ComputeFault { + qid: words[3] as u8, + stamp: words[4], + }) + } Self::CONTEXT_KILLED => { // SAFETY: Every constructor initializes all integer words. let words = unsafe { self.words }; @@ -525,3 +553,37 @@ impl ContextKill { } } } + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_g17_event_records))] +mod tests { + use super::*; + + fn record(words: &[u32]) -> EventRecord { + let mut all = [0u32; EVENT_WORDS]; + all[..words.len()].copy_from_slice(words); + EventRecord::from_words(all) + } + + #[test] + fn a_compute_fault_names_its_queue_and_stamp() { + let event = record(&[7, 1, 2, 0x39, 0x300, 0xffff_fc00, 1]).decode(); + assert_eq!(event, Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); + } + + #[test] + fn v25g83_does_not_accept_an_unqualified_fault_witness() { + use super::super::variant::InitGraph; + let fault = record(&[7, 1, 2, 0x39, 0x300]); + assert_eq!(fault.decode_for(InitGraph::G17P), + Ok(Event::ComputeFault { qid: 0x39, stamp: 0x300 })); + assert_eq!(fault.decode_for(InitGraph::V25G83), Ok(Event::Unknown(7))); + assert_eq!(record(&[7, 1, 2, 128, 0x301]).decode_for(InitGraph::V25G83), + Ok(Event::Unknown(7))); + } + + #[test] + fn an_unrecognized_type_7_layout_stays_unknown() { + assert_eq!(record(&[7, 1, 2, 128, 0x300]).decode(), Ok(Event::Unknown(7))); + assert_eq!(record(&[7, 1, 2, 9, 0x301]).decode(), Ok(Event::Unknown(7))); + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 0cb7304e30677..b1afdad816b6b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -12,7 +12,7 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; use super::queue::CommandTag; use super::{ - clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, }; @@ -507,8 +507,9 @@ impl ComputeDescriptor { const UNK_FBA: u32 = 0xe0a0_0001; const UNK_FC5: u8 = 0x9f; - /// Writes the complete descriptor. - pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { + /// Fills a descriptor cleared by `Object::initialize`. The unused firmware + /// fields and padding retain that zero initialization. + pub(crate) fn write_zeroed(&mut self, args: &ComputeArgs) -> Result { if args.cdm_end_va < args.cdm_va || (args.sampler_heap_va == 0) != (args.sampler_count == 0) || args.sampler_heap_va & 7 != 0 @@ -522,7 +523,6 @@ impl ComputeDescriptor { let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; let sampler_max = sampler_max(args.sampler_count)?; - clear(self); self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs index c237896a49fb1..f4c3ee5db22e2 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT +// SPDX-License-Identifier: GPL-2.0-only //! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). //! @@ -17,9 +17,9 @@ //! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. use super::{ - fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, RegisterMapEntry, - Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, COMPLETION_RING_INDEX, PERF_STATES, - REGISTER_MAP_SLOTS, + fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, + RegisterMapEntry, Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, + COMPLETION_RING_INDEX, PERF_STATES, REGISTER_MAP_SLOTS, }; use kernel::prelude::*; @@ -381,26 +381,10 @@ impl HwData25G83 { /// Power configuration control period, in 24 MHz clocks (8 ms). const PERIOD_CLOCKS: u32 = 200_000; -/// Highest GPU power, in mW. -pub(crate) const MAX_POWER_MW: u32 = 69_116; -/// [`MAX_POWER_MW`] as an IEEE-754 single. -const MAX_POWER_MW_F32: u32 = 0x4786_fe00; /// Temperature sensors read by the die and MTR temperature limiters. const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; -/// Leakage coefficients, IEEE-754 singles: three arrays of sixteen clusters, then four arrays -/// of two dies. Not found in the ADT; constant for the SoC. -const LEAKAGE: [u32; 56] = { - let mut words = [0; 56]; - words[0] = 0x44f0_3000; // 1921.5 - words[1] = 0x44f3_0000; // 1944.0 - words[32] = 0x4373_0000; // 243.0 - words[33] = 0x4373_0000; // 243.0 - words[52] = 0x437b_0000; // 251.0 - words[54] = 0x43bd_c000; // 379.5 - words -}; -/// Offset of [`LEAKAGE`] in the power configuration. +/// Offset of the per-chip leakage coefficients in the power configuration. const LEAKAGE_OFFSET: usize = 0x642c; /// Offset in the power configuration of the GPU address of the configuration's status (the /// configuration named by the root, `+0x80`). @@ -433,12 +417,10 @@ const POWER_CONFIG_WORDS: Words = &[ (0x9c4, 0x3f7f2e9f, 1), (0x9cc, 0x3b516154, 1), (0x9d4, 0x3ca59586, 1), - (0x9e0, MAX_POWER_MW_F32, 1), (0x9e4, 0x40a90fdb, 1), (0x9ec, 0xbc6d0cfe, 1), (0x9f0, 0x64, 1), (0x9f4, 0x3e8, 2), - (0xa04, MAX_POWER_MW, 1), (0xa38, 0x3e8, 1), (0xa70, 0x8, 1), (0xa7c, 0x3f000000, 1), @@ -448,7 +430,6 @@ const POWER_CONFIG_WORDS: Words = &[ (0xa9c, 0x3ecf2e49, 1), (0xaa8, 0x64, 1), (0xaac, 0x3e8, 1), - (0xabc, MAX_POWER_MW, 1), (0xac8, 0x10, 1), (0xad0, 0x5dc00, 1), (0xb28, 0x50, 1), @@ -508,7 +489,6 @@ const POWER_CONFIG_WORDS: Words = &[ (0x27b8, 0xc80000, 1), (0x2870, 0x3e8, 1), (0x28c0, 0x1, 1), - (0x28c8, MAX_POWER_MW, 3), (0x28e0, 0x3f800000, 1), (0x28e4, 0x4, 1), (0x28e8, 0x8, 1), @@ -521,8 +501,6 @@ const POWER_CONFIG_WORDS: Words = &[ (0x29e8, 0x3f4f2e49, 1), (0x29f4, 0x28, 1), (0x29f8, 0x3e8, 2), - (0x2a04, MAX_POWER_MW_F32, 1), - (0x2a08, MAX_POWER_MW, 1), (0x2a14, 0xf8, 1), (0x2a1c, 0x5ad200, 1), (0x2a3c, 0x3e8, 1), @@ -591,27 +569,34 @@ unsafe impl Zeroable for PowerConfig25G83 {} static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); impl PowerConfig25G83 { - /// Pointer and random words, which [`Self::hash`] leaves out. - const RUNTIME: [(usize, usize); 3] = - [(CONFIG_STATUS_POINTER, 8), (AON_POINTER, 8), (ENTROPY, 4)]; - /// Initializes the power configuration. - pub(crate) fn init(&mut self, config_va: u64, aon_va: u64, entropy: u32) { + pub(crate) fn init( + &mut self, + config_va: u64, + aon_va: u64, + entropy: u32, + max_power_mw: u32, + leakage: &[u32; 56], + ) { + self.0.fill(0); fill_words(&mut self.0, POWER_CONFIG_WORDS); - for (index, word) in LEAKAGE.iter().enumerate() { + for (index, word) in leakage.iter().enumerate() { let at = LEAKAGE_OFFSET + 4 * index; self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); } + // All copies of the power ceiling must describe the same chip. + for offset in [0xa04, 0xabc, 0x28c8, 0x28cc, 0x28d0, 0x2a08] { + self.0[offset..offset + 4].copy_from_slice(&max_power_mw.to_le_bytes()); + } + let power = crate::float::F32::from(max_power_mw).to_bits(); + for offset in [0x9e0, 0x2a04] { + self.0[offset..offset + 4].copy_from_slice(&power.to_le_bytes()); + } self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] .copy_from_slice(&(config_va + 0x80).to_le_bytes()); self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); } - - /// Hash of the contents without the pointers and the random word. - pub(crate) fn hash(&self) -> u64 { - hash_without(&self.0, &Self::RUNTIME) - } } /// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU @@ -632,50 +617,3 @@ pub(crate) const AON_TABLE: [u32; 9] = [ /// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero /// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; - -impl HwData25G83 { - /// What [`Self::hash`] leaves out: the runtime words (completion descriptors, QoS pointer) - /// and what comes from the device: the performance tables built from its operating points, - /// which are binned per device, and its enabled core count. - const UNHASHED: [(usize, usize); 10] = [ - (COMPLETION_DESCRIPTORS, 0x80), - (QOS_POINTER, 8), - (core::mem::offset_of!(Self, unk_f88), size_of::()), - (core::mem::offset_of!(Self, unk_fc0), size_of::()), - (core::mem::offset_of!(Self, perf_a), size_of::()), - (core::mem::offset_of!(Self, perf_b_freq_mhz), size_of::<[u32; PERF_STATES]>()), - (core::mem::offset_of!(Self, cs_perf), size_of::()), - (core::mem::offset_of!(Self, afr_perf), size_of::()), - (core::mem::offset_of!(Self, perf_b), size_of::()), - (core::mem::offset_of!(Self, unk_2570), size_of::()), - ]; - - /// Hash of the constant contents (everything but [`Self::UNHASHED`]). - pub(crate) fn hash(&self) -> u64 { - // SAFETY: `HwData25G83` consists of integers only and has no padding. - let bytes = unsafe { - core::slice::from_raw_parts((self as *const Self).cast::(), size_of::()) - }; - hash_without(bytes, &Self::UNHASHED) - } -} - -/// FNV-1a 64 of `bytes` with the `(offset, length)` ranges in `skip` left out. -pub(crate) fn hash_without(bytes: &[u8], skip: &[(usize, usize)]) -> u64 { - let mut hash = 0xcbf2_9ce4_8422_2325u64; - for (offset, byte) in bytes.iter().enumerate() { - if skip.iter().any(|&(at, len)| (at..at + len).contains(&offset)) { - continue; - } - hash = (hash ^ u64::from(*byte)).wrapping_mul(0x100_0000_01b3); - } - hash -} - -/// Hashes ([`hash_without`]) of the hardware data, power configuration and region C that the -/// Gravity Linux G16 encoders produce for the J616s, which this build's objects must equal. -pub(crate) mod expected { - pub(crate) const HW_DATA: u64 = 0xfd3b_a26f_570f_57cd; - pub(crate) const POWER_CONFIG: u64 = 0xb669_5b84_2ad2_36fe; - pub(crate) const REGION_C: u64 = 0xd2db_fa6b_8976_47b4; -} diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 4f3a70671eb87..ca2b6604a302b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -278,6 +278,20 @@ impl RegisterArray { self.size = (size_of_val(writes) as u32) << 16 | count as u32; Ok(()) } + + /// Stores `writes` without the entry at `skip`, exactly as [`Self::set`] stores the + /// remaining writes, without first copying them elsewhere. + pub(crate) fn set_without(&mut self, self_va: u64, writes: &[RegisterWrite], skip: usize) -> Result { + let count = writes.len().checked_sub(1).ok_or(EINVAL)?; + if skip > count || count > REGISTER_ARRAY_ENTRIES { + return Err(EINVAL); + } + self.entries[..skip].copy_from_slice(&writes[..skip]); + self.entries[skip..count].copy_from_slice(&writes[skip + 1..]); + self.self_va = self_va; + self.size = ((count * size_of::()) as u32) << 16 | count as u32; + Ok(()) + } } /// Binding of one [`RegisterArray`] to a kick entry. diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 5708a2251a062..66e7d07db48a5 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -96,8 +96,6 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, - /// [`FragmentBarrier`] (25G83 firmware). - FragmentBarrier = 4, /// [`InitBufferManager`] (25G83 firmware). InitBufferManager = 5, } @@ -569,63 +567,6 @@ impl QueueConfig { } } -/// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work -/// waits for the paired tiling stamp, in addition to the kick-entry dependency. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct FragmentBarrier { - /// [`CommandTag::FragmentBarrier`]. - pub(crate) tag: u32, - /// GPU address of the tiling queue's completion stamp word. - pub(crate) stamp_va: u64, - /// Timestamp of the tiling kick. - pub(crate) tiling_kick: u64, - /// QID of the tiling queue. - pub(crate) tiling_qid: u32, - /// Tiling stamp value waited for. - pub(crate) wait: u32, - pub(crate) unk_1c: [u8; 8], - /// Event of the wait: the tiling QID. - pub(crate) event: u32, - /// Fragment stamp value. - pub(crate) stamp_self: u32, - /// Current object ID of the fragment stage. - pub(crate) fragment_id: u64, - pub(crate) unk_34: u32, - pub(crate) unk_38: [u8; 0x10], -} - -static_assert!(size_of::() == 0x48); -static_assert!(core::mem::offset_of!(FragmentBarrier, wait) == 0x18); -static_assert!(core::mem::offset_of!(FragmentBarrier, event) == 0x24); -static_assert!(core::mem::offset_of!(FragmentBarrier, fragment_id) == 0x2c); - -impl FragmentBarrier { - /// The barrier of a pass whose tiling stage, kicked at `tiling_kick` on `tiling_qid`, - /// signals `stamp_value` at `stamp_va`. - pub(crate) fn new( - stamp_va: u64, - tiling_kick: KickTimestamp, - tiling_qid: u32, - stamp_value: u32, - fragment_id: u64, - ) -> Self { - Self { - tag: CommandTag::FragmentBarrier as u32, - stamp_va, - tiling_kick: tiling_kick.get(), - tiling_qid, - wait: stamp_value, - unk_1c: [0; 8], - event: tiling_qid, - stamp_self: stamp_value, - fragment_id, - unk_34: 1, - unk_38: [0; 0x10], - } - } -} - /// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling /// descriptor that uses a parameter buffer. The block count is read from the manager. #[repr(C, packed)] diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 0e270d40392a8..d0000282cef82 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -162,8 +162,14 @@ const UTILE_CONFIG_HEIGHT_SHIFT: u32 = 14; const UTILE_CONFIG_SIZE_MASK: u64 = 0xf000; /// The partial-render phases load and store every attached depth and stencil /// surface. -const ZLS_PARTIAL_DEPTH: u64 = 1 << 15 | 1 << 19; -const ZLS_PARTIAL_STENCIL: u64 = 1 << 14 | 1 << 18; +const ZLS_STORE_DEPTH: u64 = 1 << 19; +const ZLS_STORE_STENCIL: u64 = 1 << 18; +const ZLS_LOAD_DEPTH: u64 = 1 << 15; +const ZLS_LOAD_STENCIL: u64 = 1 << 14; +const ZLS_COMPRESS_DEPTH: u64 = 1 << 6; +const ZLS_COMPRESS_STENCIL: u64 = 1 << 8; +/// Bits of the main word a partial render sets for itself. +const ZLS_PARTIAL_CLEAR: u64 = ZLS_COMPRESS_DEPTH | ZLS_COMPRESS_STENCIL | 0xf << 56; const STENCIL_CLEAR_BASE: u32 = 0x300; /// Value of the ZLS swizzle registers and descriptor field. const ZLS_SWIZZLE: u32 = 0x0404_0404; @@ -443,6 +449,7 @@ struct PassWords { layer_mode: u64, tib_blocks: u64, partial_zls_control: u64, + partial_zls_reload: u64, aux_fb_flags: u64, bg_rsrc_spec: u64, bg_va: u64, @@ -478,12 +485,27 @@ impl PassWords { tile_config |= TILE_CONFIG_PROCESS_EMPTY_TILES; } - let mut partial_zls_control = pass.zls_control; + // The partial store keeps the main word, loads included, and stores and compresses each + // plane it has. The reload arrays add the loads, which a clearing pass doesn't have. + let mut partial_zls_control = pass.zls_control & !ZLS_PARTIAL_CLEAR; if pass.depth.va != 0 { - partial_zls_control |= ZLS_PARTIAL_DEPTH; + partial_zls_control |= ZLS_STORE_DEPTH; + } + if pass.depth.compression_va != 0 { + partial_zls_control |= ZLS_COMPRESS_DEPTH; + } + if pass.stencil.va != 0 { + partial_zls_control |= ZLS_STORE_STENCIL; + } + if pass.stencil.compression_va != 0 { + partial_zls_control |= ZLS_COMPRESS_STENCIL; + } + let mut partial_zls_reload = partial_zls_control; + if pass.depth.va != 0 { + partial_zls_reload |= ZLS_LOAD_DEPTH; } if pass.stencil.va != 0 { - partial_zls_control |= ZLS_PARTIAL_STENCIL; + partial_zls_reload |= ZLS_LOAD_STENCIL; } let mut aux_fb_flags = AUX_FB_FLAGS_BASE; @@ -511,6 +533,7 @@ impl PassWords { * samples) .div_ceil(TIB_BLOCK_SIZE), partial_zls_control, + partial_zls_reload, aux_fb_flags, bg_rsrc_spec: rsrc_spec(pass.bg.rsrc_spec), bg_va: usc(pass.bg.offset)?, @@ -636,6 +659,13 @@ fn ta_registers( ]) } +/// Tiler segment control of a two-cluster pass, what macOS writes. +const VDM_SEGMENT_CONTROL: u64 = 0x20_0440; +const VDM_SEGMENT_INDEX: u64 = 4; +/// Pages the tiler keeps out of a partial render's segments: without them the pages a pass +/// returns shrink to none and the tiler can't advance. Fifteen blocks, macOS's minimum heap. +const MIN_FREE_TVB_PAGES: u64 = 0x3c; + /// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage /// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the /// M4 Pro has run with. Returns the array and its used entry count. @@ -686,15 +716,15 @@ fn ta_registers_25g83( push(find(0x1c880, 0)?); push(RegisterWrite::new(0x1c948, gtp(4)?)); push(RegisterWrite::new(0x1c888, gtp(5)?)); - push(RegisterWrite::new(0x1c890, 0)); - push(RegisterWrite::new(0x1c918, 0)); + push(RegisterWrite::new(0x1c890, VDM_SEGMENT_CONTROL)); + push(RegisterWrite::new(0x1c918, VDM_SEGMENT_INDEX)); push(find(0x1c079, 0)?); push(find(0x1c9d8, 0)?); push(RegisterWrite::new(0x1c089, gtp2)); push(RegisterWrite::new(0x1c9e0, gtp2)); push(RegisterWrite::new(0x16c41, gtp6)); push(RegisterWrite::new(0x1ca40, gtp6)); - push(RegisterWrite::new(0x1c9a8, 0)); + push(RegisterWrite::new(0x1c9a8, MIN_FREE_TVB_PAGES)); push(RegisterWrite::new(0x1c920, gtp(7)?)); for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { push(find(reg, 0)?); @@ -930,7 +960,7 @@ fn partial_resume_registers( RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15319, w.partial_zls_reload), RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -942,13 +972,15 @@ fn partial_load_registers( [ RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), - RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + // Only the main end-of-tile program resolves, and it has to reach tiles that only + // earlier segments drew to. Without this their resolve target is never written. + RegisterWrite::new(0x10039, w.tile_config | TILE_CONFIG_PROCESS_EMPTY_TILES), RegisterWrite::new(0x15309, u64::from(p.stencil_clear | STENCIL_CLEAR_BASE)), RegisterWrite::new(0x15329, p.depth.va), RegisterWrite::new(0x153c1, p.depth.compression_va), RegisterWrite::new(0x15339, p.stencil.va), RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15319, w.partial_zls_reload), RegisterWrite::new(0x15349, u64::from(ZLS_SWIZZLE)), ] } @@ -1023,7 +1055,11 @@ pub(crate) struct TaDescriptor { /// QID of the tiling queue. pub(crate) qid: u32, pub(crate) unk_8be: u32, - pub(crate) unk_8c2: u32, + /// Non-zero on the first kick of a queue pair, like the fragment descriptor's word at the + /// same position. Set on every kick, the firmware also notifies the host of each pass's + /// tiling completion, which no retirement needs. The 25G83 firmware keeps it set on every + /// kick until this is measured there. + pub(crate) first_kick: u32, /// Completion value of the tiling kick (see /// [`KickTimestamp::completion_value`]). pub(crate) completion: u64, @@ -1072,8 +1108,6 @@ unsafe impl Zeroable for TaDescriptor {} impl TaDescriptor { const ARRAY_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; const UNK_876: u32 = u32::MAX; - const UNK_892: u32 = 1; - const UNK_8C2: u32 = 1; const UNK_932: u8 = 0x44; const UNK_93C: u8 = 1; const UNK_93E_FIRST_KICK: [u8; 2] = [0xd0, 0x91]; @@ -1110,9 +1144,7 @@ impl TaDescriptor { } else if scs_register { self.registers.set(array_va, ®isters)?; } else { - let mut trimmed = [registers[0]; TA_REGISTERS]; - trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); - self.registers.set(array_va, &trimmed)?; + self.registers.set_without(array_va, ®isters, TA_SCS_ENABLE_INDEX)?; } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; @@ -1124,12 +1156,17 @@ impl TaDescriptor { self.sampler_heap_va = pass.sampler_heap_va; self.sampler_count = pass.sampler_count; self.sampler_max = sampler_max; - self.unk_892 = Self::UNK_892; + // Asks the firmware to grow the heap. macOS keeps that for memoryless passes, which can't + // partial-render, and Honeykrisp has none. 26A428 keeps it: untested there. + self.unk_892 = u32::from(args.programs != RegisterPrograms::V25G83); self.stamp_va = args.ta.stamp_va; self.aux_stamp_va = args.ta.aux_stamp_va; self.stamp_value_2 = stamp_value; self.qid = args.ta.qid.into(); - self.unk_8c2 = Self::UNK_8C2; + self.first_kick = match args.programs == RegisterPrograms::V25G83 { + true => 1, + false => u32::from(args.ordinal == 0), + }; self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; @@ -1456,10 +1493,7 @@ impl FragmentSlot { } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { - let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; - trimmed[FRAGMENT_SCS_ENABLE_INDEX..] - .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); - desc.arrays[0].set(arrays[0], &trimmed)?; + desc.arrays[0].set_without(arrays[0], ®isters, FRAGMENT_SCS_ENABLE_INDEX)?; } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs index 2f7b598733368..03d5bfccafeca 100644 --- a/drivers/gpu/drm/asahi/g17/fw/variant.rs +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -57,19 +57,24 @@ pub(crate) struct Variant { pub(crate) few_primitives: bool, /// The firmware runs ray-traversal work (ray queries and intersection programs). pub(crate) ray_traversal: bool, + /// Tiling descriptors report whether the fragment queue still had earlier kicks + /// outstanding (`fragment_busy`). + pub(crate) fragment_busy: bool, /// Render item rings carry lead commands: InitBufferManager before the first tiling pass - /// of a buffer manager, and FragmentBarrier before every fragment descriptor. + /// of a buffer manager. pub(crate) lead_commands: bool, /// Register programs of the render descriptors. pub(crate) programs: super::render::RegisterPrograms, } -/// A firmware build a SoC supports, identified by the macOS release it ships with. +/// A firmware build a SoC supports, identified by its loaded TEXT. pub(crate) struct Build { - /// macOS version (major, minor, patch). + /// macOS version (major, minor, patch), used to order image candidates. pub(crate) os_version: [u32; 3], /// Name, for the log. pub(crate) name: &'static str, + /// The admitted loaded images, measured from the `fw-text` region. + pub(crate) images: &'static [crate::fwid::FirmwareBuild], /// Layouts and rules of the build. pub(crate) variant: &'static Variant, } @@ -78,9 +83,8 @@ pub(crate) struct Build { pub(crate) enum Firmware { /// The SoC runs a single build, which is not identified (A18 Pro). Fixed(&'static Variant), - /// One of these builds, selected by the macOS version that the boot loader reports in - /// `apple,firmware-version`: the release whose boot firmware loaded the GPU firmware, and - /// with it this firmware build. Any other version is refused. + /// One of these exact loaded images. The loader's `apple,firmware-version` is only + /// a candidate-ordering hint because multiple OS releases can share an iBoot string. Identified(&'static [Build]), } @@ -98,6 +102,7 @@ pub(crate) const V26A428: Variant = Variant { mcache: true, few_primitives: true, ray_traversal: true, + fragment_busy: true, lead_commands: false, programs: super::render::RegisterPrograms::G17P, }; @@ -121,8 +126,10 @@ pub(crate) const V25G83: Variant = Variant { mcache: false, // A fragment stage with the few-primitives flag hangs. few_primitives: false, - // A stalled ray traversal wedges the GPU without a firmware recovery. - ray_traversal: false, + ray_traversal: true, + // When a tiling pass runs out of memory with this set, the firmware resumes it without + // adding memory and the tiler stops silently; macOS always writes 0. + fragment_busy: false, lead_commands: true, programs: super::render::RegisterPrograms::V25G83, }; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 10af261aeb3ac..578bb7533f3c9 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -38,6 +38,30 @@ pub(crate) struct ComputeCompletion<'a> { offset: usize, } +/// Records in the compute completion ring that the host scans. +pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { + let slots = abi::COMPLETION_RING_SIZE / 0x40; + if slots < 0x800 { + slots + } else { + 0x800 + } +}; + +/// Timestamp bits of a completion record's start and end words. +const COMPLETION_TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; + +/// What one completion-ring slot gave a scan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CompletionFold { + /// No new observation of a pending request; the scan continues. + Passed, + /// Every request now has an observation. + Done, + /// A pending request's record whose payload the firmware is still writing. + Incomplete, +} + impl ComputeCompletion<'_> { pub(crate) fn read(&self, word: usize) -> Result { if word >= 8 { @@ -129,14 +153,6 @@ impl WorkPublication<'_> { ); } } - - fn commit(self) -> u32 { - self.copy_record(); - fence(Ordering::SeqCst); - self.producer.store(self.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - self.next - } } /// Session-long storage and mappings. Aliases are released before their backing objects. @@ -191,7 +207,13 @@ struct BuildEnv<'a> { impl BuildEnv<'_> { /// An object at a fixed offset of the kernel window. - fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { + fn fixed( + &self, + offset: u64, + size: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { KernelObject::new( self.dev, self.vm, @@ -203,7 +225,13 @@ impl BuildEnv<'_> { } /// An object placed in the dynamic part of the kernel window. - fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { + fn dynamic( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { KernelObject::new( self.dev, self.vm, @@ -308,7 +336,12 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. /// Called for a fresh row or after its free-list release was consumed. - pub(super) fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { + pub(super) fn initialize_compute_pool_blocks( + &self, + slot: u16, + page_list_va: u64, + blocks: usize, + ) -> Result { if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } @@ -394,6 +427,12 @@ impl InitData { Ok(()) } + pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { + self.compute_shared + .as_ref() + .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) + } + pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { let shared = self.compute_shared.as_ref().ok_or(EIO)?; if vm.compute_pool_mappings_match(shared.id) { @@ -510,7 +549,7 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - 2 => DataMaster::Compute, + // Compute publications reserve their slots through the port. _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; @@ -563,16 +602,6 @@ impl InitData { Ok(values) } - /// Inner queue records must be visible before this outer publication. The returned - /// producer is the retained consumer witness for this slot. - pub(super) fn publish_work( - &self, - priority: u8, - record: &super::fw::channels::WorkSlot, - ) -> Result { - Ok(self.work_publication(priority, record)?.commit()) - } - /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -671,21 +700,117 @@ impl InitData { Ok(()) } + /// Raw views of the memory publishers share without the device mutex: every work ring, + /// the completion stamps, the QoS table and the recovery state word. They stay valid for + /// this session. + pub(super) fn port_views(&self, mailbox: *const super::Mailbox) -> Result { + use super::fw::{ + channels, + queue::{DataMaster, Policy}, + }; + let ring_of = |priority: u8, engine: DataMaster| -> Result { + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); + let state = self.object(self.cluster)?; + let word = + |offset: usize| -> Result<*const AtomicU32> { Ok(state.word(base + offset)?) }; + Ok(super::port::RingView { + slots: self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel), + channels::WORK_RING_SLOTS as usize * core::mem::size_of::(), + )?, + consumers: [ + word(abi::ChannelState::CONSUMER)?, + word(abi::ChannelState::UNK_10)?, + ], + producer: word(abi::ChannelState::PRODUCER)?, + }) + }; + let rings = + |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { + let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; + for (priority, view) in views.iter_mut().enumerate().skip(1) { + *view = ring_of(priority as u8, engine)?; + } + Ok(views) + }; + Ok(super::port::Views { + compute: rings(DataMaster::Compute)?, + tiling: rings(DataMaster::Tiling)?, + fragment: rings(DataMaster::Fragment)?, + stamps: self.object(self.stamps[0])?.word(0)?, + qos: self + .qos()? + .pointer(0, core::mem::size_of::())?, + recovery: self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, + mailbox, + }) + } + /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. - pub(super) fn scan_compute_completions( + /// Folds the compute completion records the firmware appended since `start`, in ring + /// order. `seen` holds every slot's control word as of the previous pass over it: the + /// first slot still holding it is where the firmware writes next, and the pass ends there + /// instead of reading the rest of the ring. A pass also ends once `record` has found every + /// request, or at a requested record whose payload is still being written, which the next + /// pass reads again. Returns the slot the next pass starts at. + pub(super) fn fold_compute_completions( + &self, + start: usize, + seen: &mut [u64], + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, + ) -> Result { + let object = self.object(self.completion_rings[0])?; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } + fence(Ordering::Acquire); + let completion = |slot: usize| ComputeCompletion { + object, + offset: slot * 0x40, + }; + super::queue::compute::retirement::walk_completions( + start, + seen, + |slot| completion(slot).read(0), + |slot| record(completion(slot)), + ) + } + + /// Folds every slot of the compute completion ring and records its control word in + /// `seen`. For a pass that stopped before a record the firmware must already have + /// written. Returns the slot after the newest record (latest end timestamp), where the + /// firmware writes next. + pub(super) fn resync_compute_completions( &self, + seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { + ) -> Result { let object = self.object(self.completion_rings[0])?; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } fence(Ordering::Acquire); - for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { - record(ComputeCompletion { + let mut newest = None; + for (slot, seen_control) in seen.iter_mut().enumerate() { + let completion = ComputeCompletion { object, offset: slot * 0x40, - })?; + }; + let control = completion.read(0)?; + *seen_control = control; + if control != 0 { + let end = completion.read(5)? & COMPLETION_TIME_MASK; + if newest.is_none_or(|(_, latest)| end > latest) { + newest = Some((slot, end)); + } + } + record(completion)?; } - Ok(()) + Ok(newest.map_or(0, |(slot, _)| (slot + 1) % COMPUTE_COMPLETION_SLOTS)) } fn object(&self, index: usize) -> Result<&KernelObject> { @@ -709,6 +834,10 @@ impl InitData { } /// Constructs the graph after the firmware has published the high-root entries. + #[expect( + clippy::too_many_arguments, + reason = "The firmware graph builder keeps its independent input objects explicit." + )] pub(super) fn build( &mut self, dev: &AsahiDrmDevice, @@ -786,7 +915,8 @@ impl InitData { table.init(); Ok(()) })?; - let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + let page_pool = + self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(page_pool)? .initialize::(0, |table| { table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); @@ -870,7 +1000,11 @@ impl InitData { fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { use mmu::PROT_GPU_SHARED_RW as GS; use CpuMap::WriteBack as WB; - let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + let pages = self.add(KernelObject::backing( + env.dev, + low::RENDER_PAGE_LIST_SIZE, + WB, + )?)?; self.object_mut(pages)? .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { abi::fill_page_list( @@ -878,10 +1012,18 @@ impl InitData { (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), ) })?; - let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let runs = self.add(KernelObject::backing( + env.dev, + low::RENDER_RUN_LIST_SIZE, + WB, + )?)?; let first_block = self.objects.len(); for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing( + env.dev, + abi::FREE_LIST_BLOCK_SIZE, + WB, + )?)?; } self.render_pages = pages; self.render_runs = runs; @@ -889,7 +1031,12 @@ impl InitData { self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias(first_block + block, env.render, low::render_block(block), GS)?; + self.alias( + first_block + block, + env.render, + low::render_block(block), + GS, + )?; } Ok(first_block) } @@ -926,12 +1073,8 @@ impl InitData { .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; - let ready = self.add(env.fixed( - high::COMPUTE_READY_FREE_LIST, - mmu::UAT_PGSZ, - GFS, - WC, - )?)?; + let ready = + self.add(env.fixed(high::COMPUTE_READY_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(ready)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -1223,7 +1366,11 @@ impl InitData { /// The GPU's operating points: `count` of them, each with one voltage cell for the whole /// GPU or one per cluster. - fn opps(dev: &AsahiDrmDevice, count: usize, clusters: u32) -> Result> { + pub(super) fn opps( + dev: &AsahiDrmDevice, + count: usize, + clusters: u32, + ) -> Result> { let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; if opps.len() != count { return Err(EINVAL); @@ -1318,15 +1465,15 @@ impl InitData { /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). pub(super) fn init_accepted(&self) -> Result { - Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + Ok(self + .status_word(Role::Primary, 0x14)? + .load(Ordering::Acquire) + == 1) } pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { - self.object(self.cluster)?.word( - self.status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) + self.object(self.cluster)? + .word(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) } pub(super) fn status_read_words( @@ -1334,11 +1481,8 @@ impl InitData { role: Role, offset: usize, ) -> Result<[u32; N]> { - self.object(self.cluster)?.read_words( - self.status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) + self.object(self.cluster)? + .read_words(self.status_offset(role).checked_add(offset).ok_or(EINVAL)?) } pub(super) fn control_counters(&self) -> Result<[u32; 3]> { @@ -1431,13 +1575,15 @@ impl InitData { Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), }; - let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) + let end = offset + .checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { return Err(EINVAL); } let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); + + core::mem::offset_of!(abi::MainConfig, view_va) + + view * size_of::(); fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; @@ -1450,18 +1596,28 @@ impl InitData { impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { - let mask = self.object(self.cluster)? + let mask = self + .object(self.cluster)? .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot, self.recovery_slot_keys)?; - let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, - abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) - .ok().map(|words| words[0])?; + // Progress is independently available even when this firmware's key + // address is unknown. Zero keeps ownership unknown and cannot override + // firmware blame, but preserves the progress evidence for diagnostics. + let key = super::recovery::slot_key(slot, self.recovery_slot_keys).unwrap_or(0); + let monitor = self + .recovery_view_words::<6>( + abi::recovery::PROGRESS_VIEW, + abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE, + ) + .ok() + .map(|words| words[0])?; Some((key, monitor)) }); - sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .ok().map(|words| words[0]); + sources.reason = self + .recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) + .ok() + .map(|words| words[0]); Ok(sources) } @@ -1517,17 +1673,22 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entries(&self) -> Result { + fn entry(&self, index: usize) -> Result<(u32, u32)> { + if index >= super::recovery::RECOVERY_RECORDS { + return Err(EINVAL); + } let cluster = self.object(self.cluster)?; - let base = self.status_b + abi::PrimaryStatusB::RECOVERY_INFO; + let offset = self.status_b + + abi::PrimaryStatusB::RECOVERY_INFO + + index * core::mem::size_of::(); fence(Ordering::Acquire); - let mut entries = super::recovery::Entries::default(); - for index in 0..256 { - let offset = base + index * core::mem::size_of::(); - let flags = cluster.word(offset)?.load(Ordering::Relaxed); - entries.include(flags); - } - Ok(entries) + let flags = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? + .load(Ordering::Relaxed); + let stamp = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, stamp))? + .load(Ordering::Relaxed); + Ok((flags, stamp)) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs index 85aa6468fa311..670f4b96dcb76 100644 --- a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -8,7 +8,7 @@ //! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; -use crate::g17::{Config, Role}; +use crate::g17::{calibration::Calibration, Config, Role}; use crate::mmu; use abi::v25g83 as v; use core::sync::atomic::{fence, Ordering}; @@ -74,7 +74,7 @@ impl InitData { }) } - /// Writes the 25G83 firmware objects, checks them against the expected contents, then + /// Writes the 25G83 firmware objects using the chip calibration, then /// stages the opening control-ring state. fn initialize_25g83( &mut self, @@ -101,12 +101,14 @@ impl InitData { let opps = Self::opps(env.dev, cfg.opps, env.clusters)?; let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; + let calibration = Calibration::read(env.dev)?; + calibration.check_opps(&opps)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, windows: cfg.windows, reserved_slots: cfg.reserved_slots, opps: &opps, - perf: cfg.perf, + perf: &calibration.perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -117,41 +119,28 @@ impl InitData { }; // SAFETY: Plain C entropy source. let entropy = unsafe { kernel::bindings::get_random_u32() }; - let mut hashes = [0; 3]; self.object_mut(self.bundle)? .initialize::(0, |hw| { hw.init(&hw_args, &hw_args_25g83)?; - hashes[0] = hw.hash(); Ok(()) })?; self.object_mut(self.bundle)? .initialize::(v::POWER_CONFIG, |power| { - power.init(config_va, aon_va, entropy); - hashes[1] = power.hash(); + power.init( + config_va, + aon_va, + entropy, + calibration.max_power_mw, + &calibration.leakage, + ); Ok(()) })?; self.object_mut(self.region_c)? .initialize::(0, |region| { region.init(cfg.region_c); + region.max_power_mw = calibration.max_power_mw; Ok(()) })?; - hashes[2] = v::hash_without(self.object_mut(self.region_c)?.bytes_mut(), &[]); - // The constant contents must equal what the Gravity Linux encoders produce on the proven - // setup; the operating points and the core count come from the device and are not hashed. - let expected = [ - v::expected::HW_DATA, - v::expected::POWER_CONFIG, - v::expected::REGION_C, - ]; - if hashes != expected { - dev_err!( - env.dev.as_ref(), - "25G83 initialization data differs: {:#x?}, expected {:#x?}\n", - hashes, - expected - ); - return Err(EINVAL); - } for (index, word) in v::AON_TABLE.iter().enumerate() { self.object_mut(objects.aon)?.write(index * 4, *word)?; } @@ -167,7 +156,9 @@ impl InitData { v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); let cluster = self.object(self.cluster)?; cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); - cluster.dword(fwctl + 8)?.store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); + cluster + .dword(fwctl + 8)? + .store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); // One main configuration carrying both roles. let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; @@ -218,7 +209,8 @@ impl InitData { )?; for &(offset, value, repeat) in v::POWER_STATE_WORDS { for index in 0..repeat { - self.object_mut(objects.power_state)?.write(offset + 4 * index, value)?; + self.object_mut(objects.power_state)? + .write(offset + 4 * index, value)?; } } diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 61b8c3b7b4737..7e43c163ac4a7 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -14,7 +14,10 @@ use kernel::{ }; pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; -const RENDER_CREDITS: u32 = 32; +/// One logical render queue owns one physical pair, whose active ring holds +/// `RENDER_DEPTH` commands. A deeper scheduler would only park run_job in +/// admission until a retirement; the scheduler waits for the credit instead. +const RENDER_CREDITS: u32 = crate::hw::t8140::queues::RENDER_DEPTH as u32; const COMPUTE_CREDITS: u32 = 16; #[derive(Clone, Copy, PartialEq, Eq)] @@ -150,6 +153,7 @@ impl Packet { super::dependency::replay_status(fence_status(fence))?; } for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -173,8 +177,9 @@ impl Packet { kernel::time::msecs_to_jiffies(10) as isize, ) }; + // An interrupted wait is not a producer failure. if waited < 0 { - return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + return Err(Error::from_errno(waited as i32)); } if waited > 0 { break; } } @@ -183,6 +188,7 @@ impl Packet { for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -252,7 +258,7 @@ pub(crate) fn fence_status(fence: &Fence) -> i32 { fn check_status(fence: &Fence) -> Result { let status = fence_status(fence); if status < 0 { - Err(super::dependency::inherited_error(Error::from_errno(status))) + Err(super::dependency::PRODUCER_FAILED) } else { Ok(()) } @@ -262,6 +268,9 @@ fn check_status(fence: &Fence) -> Result { /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { /// Device detector retained by render completions after queue teardown. + fn input_sources(&self, _inputs: &[crate::file::SyncItem]) + -> Option> { None } + fn feed(&self) -> Option> { None } @@ -277,12 +286,16 @@ pub(crate) trait Backend: Send + Sync { fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. fn release_owner(&self); - fn prepare(&self, packet: &Packet) -> Option; + /// A fence to wait for before `publish`, signalled by the release of the capacity the + /// packet lacks. Called again once it signals. + fn prepare(&self, packet: &Arc) -> Option; fn publish(&self, packet: Arc) -> Result; fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); /// Transfer a timeout to an outstanding off-lock graph transaction, if any. fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; + /// A long job of another queue is running beside `packet`. + fn long_job_beside(&self, _packet: &Arc) -> bool { false } /// Return true only after taking publication ownership and recording its result. fn try_early(&self, packet: &Arc) -> bool; fn count_backlog(&self, packet: &Packet); @@ -297,6 +310,7 @@ struct Timeout { backend: Arc, victim_renewals: u8, execution_renewals: u8, + shared_renewals: u8, packet: Arc, } @@ -314,13 +328,14 @@ impl Drop for Job { impl sched::JobImpl for Job { type TimeoutData = Timeout; const MODULE: Option<&'static kernel::ThisModule> = - Some(&crate::THIS_MODULE); + Some(kernel::module::this_module::()); fn timeout_data(&self) -> Timeout { Timeout { backend: self.backend.clone(), victim_renewals: 0, execution_renewals: 0, + shared_renewals: 0, packet: self.packet.clone(), } } @@ -334,6 +349,13 @@ impl sched::JobImpl for Job { ) { return true; } + // Other work retiring means the GPU is shared, not hung. So does another long job + // running beside a long job, though neither retires until it ends. + let shared = super::timeout::retiring(super::recovery::since_retired(super::now_ns())) + || (job.packet.command.long_job() && job.backend.long_job_beside(&job.packet)); + if super::timeout::renew_shared(job.packet.is_published(), shared, &mut job.shared_renewals) { + return true; + } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); if deferred { job.victim_renewals = job.victim_renewals.saturating_add(1); @@ -342,10 +364,17 @@ impl sched::JobImpl for Job { } fn prepare(job: &mut Self) -> Option { - job.backend.prepare(&job.packet) + super::trace::schedule(&job.packet, 5, 0); + let wait = job.backend.prepare(&job.packet); + super::trace::schedule(&job.packet, 6, i32::from(wait.is_some())); + wait } fn run(job: &mut Self) -> Result> { - job.backend.publish(job.packet.clone()).map(Some) + super::trace::schedule(&job.packet, 3, 0); + let result = job.backend.publish(job.packet.clone()).map(Some); + super::trace::schedule(&job.packet, 4, + result.as_ref().err().map_or(0, |error| error.to_errno())); + result } fn timed_out(job: &mut Timeout, finished: bool) -> sched::Status { // The scheduler retains the detached job throughout this callback. Keep @@ -373,13 +402,32 @@ pub(crate) struct Fences { pub(crate) ready: Fence, pub(crate) completed: Fence, pub(crate) publication: Option>, + pub(crate) owner: Arc, +} + +impl super::dependency::Retention for Fences { + fn retained(&self) -> bool { + self.owner.retains_vm() + } +} + +impl Fences { + /// Scheduler::enqueue always supplies our explicit completion fence here. + /// It has no polling callback, so pending frontier checks need no lock. + pub(crate) fn status(&self) -> i32 { + unsafe extern "C" { + fn asahi_g17_completion_fence_status(fence: *mut bindings::dma_fence) -> i32; + } + // SAFETY: completed owns the live private fence throughout this read. + unsafe { asahi_g17_completion_fence_status(self.completed.raw()) } + } } impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { - Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), - Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), + Engine::Render => (RENDER_CREDITS, c_str!("aurora_g17_render")), + Engine::Compute => (COMPUTE_CREDITS, c_str!("aurora_g17_compute")), }; let scheduler = sched::Scheduler::new(dev.as_ref().as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; @@ -397,6 +445,7 @@ impl Scheduler { early: bool, publication: Option, ) -> Result { + super::trace::schedule(&packet, 0, dependencies.len() as i32); let completed = packet.completion.fence(); let host_timestamps = packet.completion.has_timestamps(); // Only commands which actually write timestamps become host-prefix @@ -429,10 +478,15 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + let inline = early && self.backend.try_early(&packet); + super::trace::schedule(&packet, if inline { 1 } else { 2 }, 0); + if !inline { + if packet.completion.needs_source_ready() { + packet.completion.set_source_ready(ready.clone()); + } self.backend.count_backlog(&packet); } job.push(); - Ok(Fences { ready, completed, publication: host_publication }) + Ok(Fences { ready, completed, publication: host_publication, owner: packet.completion.clone() }) } } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index c337a8b298f19..6499cae106309 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -24,22 +24,38 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// Minimum render-channel capacity protected from compute-only allocation. -// Protect one tiling/fragment pair for each logical execution context, plus -// the two unpublished bootstrap reservations. The remaining ten compute -// channels are shared after exact retirement, rather than pinned per client. -const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; -pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; -/// Logical queues that can each hold a render pair: what the render reserve leaves beside the -/// two bootstrap reservations once the QIDs below `floor`, which the firmware keeps, are gone -/// (every execution context with no floor). +/// Render identities that must stay reachable for `queues` logical queues: +/// one tiling/fragment pair each, plus the two bootstrap reservations. +/// +/// Installed identities never change engine before processor stop, so a +/// compute channel permanently narrows what render can still install. Compute +/// may take a free QID only while every open logical queue, and +/// [`RENDER_HEADROOM`] queues opened later, can still install a dedicated +/// pair; logical-queue admission refuses a queue whose pair no longer fits. +pub(super) const fn render_reserve(queues: usize) -> usize { + let queues = queues + RENDER_HEADROOM; + let max = crate::mmu::MAX_EXECUTION_CONTEXTS as usize; + 2 * if queues < max { queues } else { max } + 2 +} +/// Logical queues that may still open after compute growth without losing a +/// dedicated render pair. +const RENDER_HEADROOM: usize = 24; +/// The render reserve with every execution context open. +const FULL_RENDER_RESERVE: usize = render_reserve(crate::mmu::MAX_EXECUTION_CONTEXTS as usize); +/// Compute channels that stay reachable with every execution context open. +#[cfg(CONFIG_KUNIT = "y")] +pub(super) const COMPUTE_FLOOR: usize = QID_COUNT - FULL_RENDER_RESERVE; +/// Upper bound on distinct physical compute channels named by one kick. +pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT; +/// Maximum logical queues after firmware-owned QIDs are excluded. pub(super) const fn queue_limit(floor: usize) -> u32 { - ((COMPUTE_RENDER_RESERVE - 2 - floor) / 2) as u32 + ((FULL_RENDER_RESERVE - 2 - floor) / 2) as u32 } const _: () = { assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); - assert!(COMPUTE_CHANNELS >= 4); + // Compute keeps this many channels even with every execution context open. + assert!(QID_COUNT - FULL_RENDER_RESERVE >= 4); assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); }; @@ -71,7 +87,6 @@ struct Identity { pub(crate) struct QueueIds { entries: [Option; QID_COUNT], initial_render: [Option; 2], - publication: u64, leased: usize, /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). floor: usize, @@ -82,10 +97,9 @@ impl QueueIds { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), initial_render: [None; 2], - publication: 0, leased: 0, // The reserve must keep render capacity above the floor. - floor: if floor < COMPUTE_RENDER_RESERVE { floor } else { return Err(EINVAL) }, + floor: if floor < FULL_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } @@ -113,6 +127,58 @@ impl QueueIds { owner: u64, engine: DataMaster, preferred: Option, + ) -> Result { + self.reserve_for(owner, engine, preferred, crate::mmu::MAX_EXECUTION_CONTEXTS as usize) + } + + fn render_ids(&self) -> usize { + self.entries + .iter() + .flatten() + .filter(|entry| matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment)) + .count() + } + + /// Whether `queues` open logical queues can each still install a dedicated + /// render pair, beside the bootstrap reservations and installed compute. + pub(crate) fn render_fits(&self, queues: usize) -> bool { + self.render_ids() + (QID_COUNT - self.floor - self.leased) >= 2 * queues + 2 + } + + /// Whether a render pair not yet built can still reserve both of its identities. + pub(crate) fn render_pair_fits(&self) -> bool { + QID_COUNT - self.floor - self.leased >= 2 + } + + /// Whether a fresh compute identity fits beside the render reserve of + /// `queues` open logical queues, without reserving it. + pub(crate) fn compute_fits(&self, queues: usize) -> bool { + let initial = self.leased == QID_COUNT - self.floor + && self.initial_render.iter().flatten().any(|id| self.can_cancel_unpublished(*id)); + self.compute_room(queues, usize::from(initial)) + } + + fn compute_room(&self, queues: usize, replacement: usize) -> bool { + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let render = self.render_ids(); + QID_COUNT - self.floor - self.leased + replacement + > render_reserve(queues) + .min(FULL_RENDER_RESERVE - self.floor) + .saturating_sub(render - replacement) + } + + /// As [`Self::reserve`], with the compute reserve sized for `queues` open + /// logical queues instead of every possible execution context. + pub(crate) fn reserve_for( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + queues: usize, ) -> Result { if owner == 0 { return Err(EINVAL); @@ -129,10 +195,8 @@ impl QueueIds { Err(EINVAL) }; } - // The QIDs below the floor are never the host's; the reserve shrinks - // with them, so the compute channel count stays the same. + // The QIDs below the floor are never available to the host. let total = QID_COUNT - self.floor; - let reserve = COMPUTE_RENDER_RESERVE - self.floor; let initial = if engine == DataMaster::Compute && self.leased == total { self.initial_render .iter() @@ -145,27 +209,9 @@ impl QueueIds { } else { None }; - // Existing render identities already satisfy this reservation. Counting - // them twice can refuse compute even while physical QIDs remain free. // Published identities always retain their original lifetime rules. - if engine == DataMaster::Compute { - let render = self - .entries - .iter() - .flatten() - .filter(|entry| { - matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) - }) - .count(); - // Replacing an initial reservation consumes one render identity. - // Include that transition when protecting render's minimum, and - // count the replacement as available compute capacity. - let replacement = usize::from(initial.is_some()); - if total - self.leased + replacement - <= reserve.saturating_sub(render - replacement) - { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); - } + if engine == DataMaster::Compute && !self.compute_room(queues, usize::from(initial.is_some())) { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); } let index = preferred .map(|index| usize::from(index) + self.floor) @@ -196,13 +242,6 @@ impl QueueIds { Ok(id) } - /// Ordered under the device mutex; no atomic operation on the submission path. Compute and - /// render publications share it, so a watchdog can order work across both engines. - pub(crate) fn next_publication(&mut self) -> Result { - self.publication = self.publication.checked_add(1).ok_or(EOVERFLOW)?; - Ok(self.publication) - } - /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { @@ -249,10 +288,10 @@ mod accounting_tests { #[test] fn render_identities_satisfy_the_compute_reserve() { let mut ids = table(); - for owner in 1..=COMPUTE_RENDER_RESERVE as u64 { + for owner in 1..=FULL_RENDER_RESERVE as u64 { ids.reserve(owner, DataMaster::Tiling, None).unwrap(); } - for owner in COMPUTE_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { + for owner in FULL_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { ids.reserve(owner, DataMaster::Compute, None).unwrap(); } assert_eq!(ids.leased, QID_COUNT); @@ -263,14 +302,49 @@ mod accounting_tests { #[test] fn compute_cannot_consume_the_render_minimum() { let mut ids = table(); - for owner in 1..=(QID_COUNT - COMPUTE_RENDER_RESERVE) as u64 { + for owner in 1..=(QID_COUNT - FULL_RENDER_RESERVE) as u64 { ids.reserve(owner, DataMaster::Compute, None).unwrap(); } assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); ids.cancel_unpublished(tiling).unwrap(); - assert_eq!(ids.leased, QID_COUNT - COMPUTE_RENDER_RESERVE); + assert_eq!(ids.leased, QID_COUNT - FULL_RENDER_RESERVE); + } + + #[test] + fn dynamic_compute_growth_preserves_render_headroom_above_firmware_floor() { + for floor in [0, 4, 32] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = 2; + let mut owner = 1; + while ids.compute_fits(queues) { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + owner += 1; + } + assert_eq!(ids.leased, QID_COUNT - floor - render_reserve(queues)); + assert_eq!(ids.reserve_for(owner, DataMaster::Compute, None, queues), Err(ENOSPC)); + assert!(ids.render_fits(queues + RENDER_HEADROOM)); + assert!(!ids.render_fits(queues + RENDER_HEADROOM + 1)); + } + } + + #[test] + fn full_logical_queue_budget_keeps_compute_floor_and_excludes_firmware_ids() { + for floor in [0, 4, 32] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = queue_limit(floor) as usize; + for owner in 1..=COMPUTE_FLOOR as u64 { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + } + assert!(!ids.compute_fits(queues)); + assert!(ids.render_fits(queues)); + assert!(!ids.render_fits(queues + 1)); + } } #[test] diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 2999daec47015..1dbfcc8fd3ce5 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -12,12 +12,18 @@ //! //! # Locking //! -//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all -//! firmware-shared state, including the bounded waits for the firmware that publication needs. -//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state -//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. - +//! `Shared::state` is the device lock. It serializes binding, recovery, the event worker and all +//! firmware-shared state that has no narrower owner. Steady-state compute publication into a +//! bound queue takes only that queue's cell lock and the port's locks ([`port`]), reached under +//! a preparation lease. Completions are signalled after dropping the device lock. Lock order: +//! `Shared::state` -> compute queue cell (one at a time) -> port locks -> availability domains; +//! `Shared::state` -> UAT shared state -> VM exec lock -> GEM reservation. `RoleState::ack` +//! nests inside nothing. Capacity waits block on an availability domain signalled by the +//! release that provides the capacity, never on a sleep. + +mod availability; mod buffer; +mod calibration; mod channel; mod command; pub(crate) use command::Attachments; @@ -25,29 +31,36 @@ mod completion; pub(crate) mod context; mod data_segment; mod dependency; +mod sync_input; +pub(crate) mod sync_output; +mod trace; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; +mod capacity; mod event; mod feed; mod fence; mod freelist; pub(crate) mod fw; -mod ids; pub(crate) mod hazard; +mod ids; mod initdata; mod job; -mod capacity; mod kick; mod object; +mod port; pub(crate) mod power; mod preparation; mod qos; mod queue; mod recovery; mod replay; +mod reports; mod runtime; +pub(crate) mod stats; pub(crate) mod status; mod teardown; +pub(crate) mod telemetry; mod timeout; mod validation; @@ -55,6 +68,8 @@ use core::any::Any; use core::ops::Range; use core::sync::atomic::{ AtomicBool, + AtomicU32, + AtomicU64, Ordering, // }; @@ -62,12 +77,12 @@ use kernel::{ c_str, device::{self, Core}, drm::gem::shmem, + impl_has_delayed_work, impl_has_work, io::{ IoBase, IoSysMap, Region, // }, - impl_has_delayed_work, impl_has_work, new_condvar, new_delayed_work, new_mutex, new_work, platform, prelude::*, soc::apple::rtkit, @@ -192,9 +207,10 @@ pub(crate) struct Config { /// Layouts of the firmware command buffers that follow a SoC's GPU core layout. pub(crate) struct CoreLayout { - /// The cluster and enabled core counts the layouts were measured on, which the GPU must - /// report at probe; `None` where the layouts hold for every configuration of the SoC. + /// The measured cluster and enabled core counts; `None` admits every configuration. pub(crate) cores: Option<(u32, u32)>, + /// An experimental layout: core slots and enabled cores per cluster. + pub(crate) alternate_core_masks: Option<(u32, u32)>, /// A compute queue's operand-state buffer. pub(crate) operand_state: fw::compute::OperandState, /// A render's deflake buffer. @@ -359,7 +375,7 @@ impl rtkit::Operations for RoleOps { if !is_events_message(ep, msg) { return false; } - data.shared.queue_events(); + data.shared.queue_event_records(); true } @@ -394,13 +410,58 @@ impl rtkit::Operations for RoleOps { } } +/// The RTKit of one coprocessor. The mailbox driver serializes its own register writes; this +/// lock only serializes the exclusive borrows the RTKit API takes, so that publishers holding +/// different locks can ring the same doorbell. +#[pin_data] +struct Mailbox { + #[pin] + lock: Mutex<()>, + rtkit: core::cell::UnsafeCell>, +} + +// SAFETY: Every access to the RTKit goes through `with`, which holds the lock. +unsafe impl Sync for Mailbox {} + +impl Mailbox { + fn new(rtkit: rtkit::RtKit) -> Result>> { + KBox::pin_init( + pin_init!(Self { + lock <- new_mutex!((), "g17::Mailbox::lock"), + rtkit: core::cell::UnsafeCell::new(rtkit), + }), + GFP_KERNEL, + ) + } + + fn with(&self, f: impl FnOnce(Pin<&mut rtkit::RtKit>) -> R) -> R { + let _guard = self.lock.lock(); + // SAFETY: The lock excludes every other borrow of the RTKit. + f(Pin::new(unsafe { &mut *self.rtkit.get() })) + } + + fn send(&self, endpoint: u8, message: u64) -> Result { + self.with(|rtkit| rtkit.send_message(endpoint, message)) + } + + /// A handle on the receive thread's minimum utilization, usable from any context. + /// + /// # Safety + /// + /// The caller must not use the handle after this mailbox is dropped. + unsafe fn rx_util_min_handle(&self) -> rtkit::RxUtilMin { + // SAFETY: The caller's obligation is the handle's. + self.with(|rtkit| unsafe { rtkit.rx_util_min_handle() }) + } +} + /// A running firmware coprocessor. /// /// Dropping it stops the CPU before RTKit and the crash log buffer go away. struct Coprocessor { cpu: regs::CpuControl, running: bool, - rtkit: rtkit::RtKit, + rtkit: Pin>, state: Arc, _crash_buffer: Option, } @@ -455,7 +516,7 @@ impl Coprocessor { Ok(Coprocessor { cpu: regs::CpuControl::new(pdev, index)?, running: false, - rtkit: Self::receiver(dev, index, &state)?, + rtkit: Mailbox::new(Self::receiver(dev, index, &state)?)?, state, _crash_buffer: crash_buffer, }) @@ -484,19 +545,20 @@ impl Coprocessor { fn boot(&mut self) -> Result { self.cpu.start()?; self.running = true; - Pin::new(&mut self.rtkit).boot() + self.rtkit.with(|rtkit| rtkit.boot()) } /// Starts the firmware endpoints and hands the firmware its initialization root. An /// acknowledgment message is awaited here; one in status A is the caller's to poll. fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { - let mut rtkit = Pin::new(&mut self.rtkit); - rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; - rtkit.as_mut().start_endpoint(EP_DOORBELL)?; - rtkit.send_message( - EP_FIRMWARE, - MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), - )?; + self.rtkit.with(|mut rtkit| { + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + ) + })?; match ack { fw::variant::InitAck::Message => self.state.wait_for_ack(), fw::variant::InitAck::Status => Ok(()), @@ -504,8 +566,9 @@ impl Coprocessor { } /// Sends a host notification to the firmware. - fn notify(&mut self, msg: u64) -> Result { - Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) + fn notify(&self, msg: u64) -> Result { + trace::message(trace::MAILBOX, msg); + self.rtkit.send(EP_DOORBELL, msg) } /// Stops the CPU. The firmware keeps its state, but no longer runs. @@ -572,6 +635,10 @@ struct Firmware { secondary: Option, primary: Coprocessor, bridge: Bridge, + /// Publication state shared with publishers that do not hold the device mutex. + port: Arc, + /// Fast-path submissions already accounted by the idle check. + seen_submissions: u64, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, @@ -595,6 +662,7 @@ impl Drop for Firmware { secondary.stop(); } self.primary.stop(); + self.port.set_effort_full(false); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while // those references still keep the register block powered. @@ -641,42 +709,71 @@ impl Firmware { /// Called at actual submission publication, not while a queued job is merely being built. fn note_submission(&mut self) -> Result { + self.port.note_gpu_activity(); self.effort.submit(); self.apply_effort(power::FULL_EFFORT) } fn apply_effort(&mut self, effort: u32) -> Result { - Self::apply_shared_effort(&self.init, &mut self.effort, effort) + Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) } fn apply_shared_effort( init: &initdata::InitData, state: &mut power::Effort, + port: &port::Port, effort: u32, ) -> Result { if state.applied() == effort && init.effort()? == effort { return Ok(()); } init.publish_effort(effort)?; + trace::message(trace::EFFORT, u64::from(effort)); state.did_apply(effort); + port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) } /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, - /// unretired work; quarantined work retaining mappings is excluded. + /// unretired work; quarantined work retaining mappings is excluded. Submissions published + /// without the device mutex count as activity, and a clear that races one is abandoned. + fn observe_statistics_power(&mut self) { + if self.rings.statistics_suspended() { + return; + } + self.rings.observe_statistics_power( + self.bridge.regs.g17_core_power() & 0xf != 0, + self.port.statistics_epoch(), + ); + } + fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - let effort = match self.effort.idle(now_ns, in_flight) { + let submissions = self.port.submissions(); + let submitted = submissions != self.seen_submissions; + self.seen_submissions = submissions; + if submitted { + self.effort.submit(); + } + let effort = match self.effort.idle(now_ns, in_flight || submitted) { power::IdleAction::None => None, power::IdleAction::Clear => { + self.port.set_effort_full(false); + if self.port.submissions() != submissions { + self.port.set_effort_full(true); + self.effort.submit(); + return Ok(Some(power::IDLE_HOLD_NS)); + } self.apply_effort(0)?; None } power::IdleAction::Recheck(delay) => Some(delay), }; - Ok(match (effort, self.idle_hold(now_ns, in_flight)) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - }) + Ok( + match (effort, self.idle_hold(now_ns, in_flight || submitted)) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }, + ) } /// Drives the idle hold (see power::Hold); returns the delay in ns for its next check. A @@ -685,7 +782,10 @@ impl Firmware { const RETRY_NS: u64 = 1_000_000; let hold_ms = power::IDLE_HOLD_MS.load(Ordering::Relaxed); for _ in 0..2 { - match self.hold.step(now_ns, in_flight, self.effort.submissions(), hold_ms) { + match self + .hold + .step(now_ns, in_flight, self.effort.submissions(), hold_ms) + { power::HoldAction::None => return None, power::HoldAction::Recheck(delay) => return Some(delay), power::HoldAction::Assert => { @@ -773,8 +873,28 @@ struct Shared { #[pin] feed_work: Work, preparations: Arc, + /// Compute admission waiting for a physical channel: signalled when a channel becomes + /// reusable or shareable, a QID or slot is returned, or a pool vacancy is witnessed. + channels: Arc, + /// Compute admission waiting for retirement-bound capacity (work states, ring and + /// in-flight room, drain witnesses, pool releases) or for publication to reopen. + progress: Arc, + /// Logical-queue creation waiting for a returned queue slot or execution context. + queues: Arc, + /// Render publication waiting for render capacity, a pair on loan, a work + /// state or firmware room: signalled when a pair, lease or work state + /// returns, an owner closes, a teardown parks, and on every event pass + /// while a render publisher waits (firmware consumption has no event). + renders: Arc, + /// Unbound compute publishers which can reuse a retired channel. + compute_demand: AtomicU32, #[pin] work: Work, + /// The next event pass runs every housekeeping step: set by every [`Self::queue_events`] + /// request, not by the firmware's event notification ([`Self::queue_event_records`]). + housekeeping: AtomicBool, + /// When an event pass last ran every housekeeping step, in ns. + housekeeping_ns: AtomicU64, #[pin] poll: DelayedWork, #[pin] @@ -795,6 +915,8 @@ struct Shared { state: Mutex>>, #[pin] changed: CondVar, + /// The port of the session in `state`; null before boot and once detachment begins. + port: core::sync::atomic::AtomicPtr, } impl_has_work! { @@ -884,8 +1006,37 @@ impl Shared { drop(retired); } - /// Queues the event worker. Callable from any context. + /// Wakes the waiters for a returned pair, binding or capacity change. + fn note_changed(&self) { + self.changed.notify_all(); + self.renders.notify(); + } + + /// The session's port, valid while `lease` is held: shutdown and device loss join every + /// lease before they detach the session. + fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { + let _ = lease; + // SAFETY: The pointer is published before the session becomes reachable and cleared + // only after every lease has been joined, so it names a live port for 'a. + unsafe { self.port.load(Ordering::Acquire).as_ref() } + } + + /// Called after joining every preparation lease, before the session is detached. + fn forget_port(&self) { + self.port.store(core::ptr::null_mut(), Ordering::Release); + } + + /// Queues the event worker for a pass that runs every housekeeping step. Callable from any + /// context. fn queue_events(self: &Arc) { + self.housekeeping.store(true, Ordering::Release); + self.queue_event_records(); + } + + /// Queues the event worker for firmware records alone: completions, or records that arrived + /// during a pass. The pass skips the housekeeping that nothing needs (see + /// `runtime_event_turn`). Callable from any context. + fn queue_event_records(self: &Arc) { // Already queued if this fails, which is just as good. let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); } @@ -912,6 +1063,7 @@ impl Shared { // Context release returns its execution root off-lock. The next // device pass can now detach idle compute aliases and their QoS // leases, without depending on a new client's submission/event. + self.queues.notify(); self.queue_events(); } } @@ -991,7 +1143,10 @@ pub(crate) struct Gpu { geometry: mmu::UatGeometry, ids: gpu::SequenceIDs, shared: Arc, + stats: Arc, _capacity: Pin>>, + _reports: Pin>>, + _power: Pin>>, } impl Gpu { @@ -1002,8 +1157,22 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + let calibration = if cfg.chip_id == crate::hw::t6040::CHIP_ID { + Some(calibration::Calibration::read(dev).inspect_err(|_| { + dev_err!( + dev.as_ref(), + "Missing or invalid T6040 GPU calibration handoff\n" + ); + })?) + } else { + None + }; let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; + if let Some(calibration) = calibration { + calibration.check_cores(&id.core_masks)?; + calibration.check_opps(&initdata::InitData::opps(dev, cfg.opps, id.num_clusters)?)?; + } if id.family != cfg.identity.family || id.variant != cfg.identity.variant || id.num_dies != cfg.identity.num_dies @@ -1023,14 +1192,37 @@ impl Gpu { oas: cfg.uat_oas, }; let max_frequency_khz = Self::max_frequency_khz(dev)?; - let preparations = preparation::Gate::new()?; + let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; + let idle_delay = + power::idle_delay(node.get_opt_property(c_str!("apple,idle-off-delay-ms"))?)?; + power::IDLE_HOLD_MS.store(idle_delay, Ordering::Relaxed); + dev_info!( + dev.as_ref(), + "GPU idle power-off delay: {} ms\n", + idle_delay + ); + let channels = availability::Availability::new()?; + let progress = availability::Availability::new()?; + let queues = availability::Availability::new()?; + let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; + let renders = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + let alternate = cfg.core_layout.cores.is_some_and(|(clusters, cores)| { + clusters == id.num_clusters + && cores != active_cores + && cfg.core_layout.alternate_core_masks.is_some_and(|(slots, enabled)| { + id.num_cores == slots + && id.core_masks.len() == clusters as usize + && id.core_masks.iter().all(|mask| mask.count_ones() == enabled) + }) + }); if cfg .core_layout .cores .is_some_and(|cores| cores != (id.num_clusters, active_cores)) + && !alternate { dev_err!( dev.as_ref(), @@ -1040,6 +1232,12 @@ impl Gpu { ); return Err(ENODEV); } + if alternate { + dev_warn!( + dev.as_ref(), + "Experimental GPU topology admission: command-buffer offsets are not yet validated\n" + ); + } let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), @@ -1054,7 +1252,14 @@ impl Gpu { feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, + channels, + progress, + queues, + renders, + compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), + housekeeping: AtomicBool::new(true), + housekeeping_ns: AtomicU64::new(0), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), grow <- new_work!("g17::Shared::grow"), @@ -1063,15 +1268,31 @@ impl Gpu { stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), + port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), GFP_KERNEL, )?; - let directory = kernel::debugfs::Dir::new(c"asahi-g17"); + let directory = kernel::debugfs::Dir::new(c"aurora-g17"); let capacity = KBox::pin_init( directory.read_only_file(c"capacity", capacity::View(shared.clone())), GFP_KERNEL, )?; + let reports = KBox::pin_init( + directory.read_only_file(c"firmware_reports", reports::View(shared.clone())), + GFP_KERNEL, + )?; + let power = KBox::pin_init( + directory.read_only_file(c"power", reports::PowerView(shared.clone())), + GFP_KERNEL, + )?; + let stats = Arc::new( + crate::stats::StatsSnapshot { + g17_reader: Some(stats::Reader(shared.clone())), + ..Default::default() + }, + GFP_KERNEL, + )?; let gpu = Arc::new( Gpu { cfg, @@ -1080,11 +1301,17 @@ impl Gpu { geometry, ids: Default::default(), shared, + stats, _capacity: capacity, + _reports: reports, + _power: power, }, GFP_KERNEL, )?; let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + gpu.shared + .port + .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); *gpu.shared.state.lock() = Some(firmware); gpu.shared.queue_events(); Ok(gpu) @@ -1101,21 +1328,43 @@ impl Gpu { fw::variant::Firmware::Identified(builds) => builds, }; let version = Self::os_firmware_version(dev); - if let Some(build) = builds.iter().find(|build| Some(build.os_version) == version) { - dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); - return Ok(build.variant); + // iBoot strings can be shared by OS releases. Prefer the reported release's + // candidate, but admit only an exact loaded image, regardless of that hint. + let preferred = builds + .iter() + .filter(|build| Some(build.os_version) == version); + let others = builds + .iter() + .filter(|build| Some(build.os_version) != version); + for build in Iterator::chain(preferred, others) { + if Self::identify_image(dev, build).is_ok() { + dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + return Ok(build.variant); + } } dev_err!( dev.as_ref(), - "Unsupported GPU firmware: macOS version {:?} (apple,firmware-version)\n", + "Unsupported GPU firmware image (reported macOS version {:?})\n", version ); - for build in builds.iter() { - dev_err!(dev.as_ref(), "Supported GPU firmware: {}\n", build.name); - } Err(ENODEV) } + /// Checks that the loaded TEXT (the `fw-text` region) is one of `build`'s images, with the DATA size + /// of the `fw-data` region. The version property is only a selection hint. + fn identify_image(dev: &AsahiDrmDevice, build: &'static fw::variant::Build) -> Result { + let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let text = node.reserved_mem_region_to_resource_byname(c_str!("fw-text"))?; + let data = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; + let start = text.start(); + let end = start.checked_add(text.size()).ok_or(EINVAL)?; + crate::fwid::identify(dev.as_ref().as_ref(), start..end, data.size(), build.images) + .inspect_err(|e| { + dev_err!(dev.as_ref(), "GPU firmware image not admitted: {:?}\n", e) + })?; + Ok(()) + } + /// The macOS version the boot loader reports for the boot firmware that loaded the GPU /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { @@ -1172,13 +1421,21 @@ impl Gpu { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let primary = + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e) + })?; let secondary = match cfg.secondary_root_offset { Some(_) => Some( - Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { - dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) - })?, + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err( + |e| { + dev_err!( + dev.as_ref(), + "Secondary coprocessor setup failed: {:?}\n", + e + ) + }, + )?, ), None => None, }; @@ -1196,17 +1453,24 @@ impl Gpu { secondary, primary, bridge, - rings: channel::Rings::new(), + port: port::Port::new( + power::Demand::new(pdev.as_ref().as_raw())?, + shared.feed.clone(), + )?, + seen_submissions: 0, + rings: channel::Rings::new(shared.variant.graph)?, recovery: recovery::State::new(), effort: power::Effort::new(), hold: power::Hold::new(), registration_power_release_pending: false, variant: shared.variant, - queues <- runtime::Registry::new( + queues <- runtime::Registry::with_waiters( shared.clusters, shared.deflake, shared.descriptor_flags, shared.variant, + Some(shared.queues.clone()), + Some(shared.renders.clone()), ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, @@ -1242,12 +1506,19 @@ impl Gpu { if let Some(root_entries) = root_entries { session.uat.confirm_firmware_root_entries(root_entries)?; } + let mailbox: *const Mailbox = &*session.primary.rtkit; + session.port.attach( + session.init.port_views(mailbox)?, + session.bridge.regs.clone(), + )?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; } let ack = shared.variant.init_ack; - session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + session + .primary + .init(session.init.root_va(Role::Primary)?, ack)?; if ack == fw::variant::InitAck::Status { Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; } @@ -1260,7 +1531,12 @@ impl Gpu { secondary.notify(MSG_POWER_ON_DONE)?; } } - Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; + Self::wait_for_opening( + dev, + &session.init, + &session.primary, + session.secondary.as_ref(), + )?; session .uat .activate_render_context(&session._render_global)?; @@ -1353,7 +1629,7 @@ impl gpu::Gpu for Gpu { } fn stats_snapshot(&self) -> Option> { - None + Some(self.stats.clone()) } fn manager(&self) -> Option<&dyn gpu::GpuManager> { @@ -1377,11 +1653,11 @@ impl gpu::Gpu for Gpu { u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD) + | sync_output::FEATURE_CUMULATIVE_SIGNALS; let variant = self.shared.variant; if variant.few_primitives { - params.features |= - u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); } if variant.ray_traversal { params.features |= @@ -1463,6 +1739,7 @@ impl gpu::Gpu for Gpu { _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); @@ -1475,16 +1752,25 @@ impl gpu::Gpu for Gpu { let policy = scheduling_policy(priority)?; let admission_started = now_ns(); let (backend, slot) = loop { + let observed = self.shared.queues.observe(); match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, Err(error) if error == ENOSPC => { // Closed logical queues can still own execution/QoS IDs // until the scheduler release is consumed and reclaimed. // Backend construction has rolled back all temporary leases. - if now_ns().saturating_sub(admission_started) - >= job::TIMEOUT_MS as u64 * 1_000_000 { return Err(EBUSY); } + let waited = now_ns().saturating_sub(admission_started); + let limit = u64::from(job::TIMEOUT_MS) * 1_000_000; + if waited >= limit { + return Err(EBUSY); + } self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + // A returned queue slot or reclaimed context wakes this. + self.shared.queues.block( + observed, + (limit - waited).div_ceil(1_000_000) as u32, + true, + )?; } result => break result?, } @@ -1495,6 +1781,7 @@ impl gpu::Gpu for Gpu { backend.clone(), frontend, slot, + usage, ) { Ok(frontend) => frontend, Err(error) => { @@ -1510,18 +1797,22 @@ impl gpu::Gpu for Gpu { mut bo: gem::ObjectRef, range: Range, ) -> Result { - let state = self.shared.state.lock(); - let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - crate::timestamp::Buffer::firmware(bo.map_range_into_range( - firmware.uat.kernel_vm(), - range, - addresses, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - )?) + let mapping = { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + )? + }; + // The buffer's CPU mapping takes the reservation lock: not under the device mutex. + crate::timestamp::Buffer::firmware(mapping) } } diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs index f1b22ad5d7331..87fdb0e8a62ad 100644 --- a/drivers/gpu/drm/asahi/g17/object.rs +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -466,8 +466,17 @@ struct PoolInner { reserved: [usize; POOL_FAMILIES], bytes: usize, stopped: bool, + /// Objects the firmware may still reference after their owner dropped them, kept mapped + /// and out of reuse until firmware stop (see [`PooledObject::retain_until_stop`]). + kept: KVec, + kept_bytes: usize, + /// Kept objects still owned by their users, whose list entries are reserved. + kept_reserved: usize, } +/// Retained objects past this many are still retained, with one warning. +const KEPT_WARN: usize = 256; + /// Per-device cache-safe object recycling. Every converted mapping reserves /// one return slot before its page-table attributes change. Active leases and /// poisoned mappings count against the same capacity as reusable objects. @@ -490,6 +499,9 @@ struct CacheLease { pub(crate) struct PooledObject { object: core::mem::ManuallyDrop, lease: Option, + /// The pool keeping this object until firmware stop, and whether `reserve_kept` took its + /// list entry. + keep: Option<(kernel::sync::Arc, bool)>, } impl core::ops::Deref for PooledObject { @@ -503,11 +515,28 @@ impl core::ops::DerefMut for PooledObject { &mut self.object } } +impl PooledObject { + /// The firmware may still reference this object after its owner drops it: keep it mapped + /// and never hand it out again before firmware stop. It leaves the reuse cache, so it + /// does not take cache capacity. Call from sleepable context: this reserves the list + /// entry, so the drop, wherever it happens, does not allocate. + pub(crate) fn retain_until_stop(&mut self, pool: &kernel::sync::Arc) { + if self.keep.is_none() { + let reserved = pool.reserve_kept(); + self.keep = Some((pool.clone(), reserved)); + } + } +} + impl Drop for PooledObject { fn drop(&mut self) { // SAFETY: This destructor is the only operation that moves `object` // out; ManuallyDrop prevents a second implicit drop of the field. let object = unsafe { core::mem::ManuallyDrop::take(&mut self.object) }; + if let Some((pool, reserved)) = self.keep.take() { + pool.keep_object(object, self.lease.take(), reserved); + return; + } match self.lease.take() { Some(lease) => lease.pool.return_object(object, &lease), None => drop(object), @@ -523,7 +552,10 @@ impl Pool { } kernel::sync::Arc::pin_init( pin_init!(Self { - inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false }, "g17::object::Pool"), + inner <- kernel::new_mutex!(PoolInner { + returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false, kept: KVec::new(), + kept_bytes: 0, kept_reserved: 0, + }, "g17::object::Pool"), }), GFP_KERNEL, ) @@ -572,6 +604,7 @@ impl Pool { return Ok(PooledObject { object: core::mem::ManuallyDrop::new(entry.object), lease: Some(lease()), + keep: None, }); } let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?; @@ -596,6 +629,7 @@ impl Pool { let mut result = PooledObject { object: core::mem::ManuallyDrop::new(object), lease: reserved.then(lease), + keep: None, }; if !reserved { return Ok(result); @@ -655,10 +689,67 @@ impl Pool { core::mem::forget(object); } + /// Room for one more kept object, taken by the matching `keep_object`. Returns whether + /// it was reserved; after a failed allocation that drop leaks the object instead. + fn reserve_kept(&self) -> bool { + let mut inner = self.inner.lock(); + let pending = inner.kept_reserved + 1; + if inner.kept.capacity() - inner.kept.len() < pending + && inner.kept.reserve(pending, GFP_KERNEL).is_err() + { + return false; + } + inner.kept_reserved = pending; + true + } + + fn keep_object(&self, object: KernelObject, lease: Option, reserved: bool) { + let mut inner = self.inner.lock(); + if reserved { + inner.kept_reserved = inner.kept_reserved.saturating_sub(1); + } + if let Some(lease) = &lease { + // The object leaves the reuse cache and its capacity. + inner.reserved[lease.family as usize] -= 1; + inner.bytes -= object.size(); + } + if inner.stopped { + drop(inner); + drop(object); + return; + } + let size = object.size(); + if inner.kept.len() == KEPT_WARN { + pr_warn!("G17 keeps {} firmware-linked objects until the GPU firmware restarts\n", KEPT_WARN); + } + if !reserved { + // Another kept object's reservation must not be used: leak this one instead. + pr_err!("G17 could not record a firmware-linked object; leaking it\n"); + core::mem::forget(object); + return; + } + // retain_until_stop reserved the entry; this never allocates. + match inner.kept.push_within_capacity(object) { + Ok(()) => inner.kept_bytes += size, + Err(error) => { + // Never unmap an object the firmware may still reference. + pr_err!("G17 could not record a firmware-linked object; leaking it\n"); + core::mem::forget(error.0); + } + } + } + /// Releases retained mappings after both firmware processors have stopped. /// Active objects returned later observe the same stop witness. pub(crate) fn stop_after_firmware(&self) { - self.inner.lock().stopped = true; + let kept = { + let mut inner = self.inner.lock(); + inner.stopped = true; + inner.kept_bytes = 0; + core::mem::take(&mut inner.kept) + }; + // Mapping teardown must not run while the pool mutex is held. + drop(kept); for index in 0..POOL_FAMILIES * POOL_SLOTS { let entry = { let mut inner = self.inner.lock(); @@ -684,6 +775,7 @@ impl Pool { inner.reserved[family], POOL_SLOTS, entries.iter().flatten().count(), entries.iter().flatten().filter(|entry| entry.poisoned).count())?; } - writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES) + writeln!(f, "cached_object_bytes used={} total={}", inner.bytes, POOL_BYTES)?; + writeln!(f, "firmware_linked_objects kept={} bytes={}", inner.kept.len(), inner.kept_bytes) } } diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 7fef15084c50a..a2e293df91510 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -6,16 +6,28 @@ use core::sync::atomic::AtomicU32; pub(super) const FULL_EFFORT: u32 = 0x10000; -const IDLE_HOLD_NS: u64 = 16_000_000; +pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; /// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after /// the last job. Not measured on other firmware; at or below it the hold stays off. pub(crate) const FIRMWARE_IDLE_OFF_MS: u32 = 2; -/// The `idle_off_delay_ms` sysfs knob. Above [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU -/// cores powered until the GPU has been idle this long ([`Hold`]). +/// Idle policy initialized from the existing DT `apple,idle-off-delay-ms` property. +/// The existing `idle_off_delay_ms` sysfs attribute may override it. Above +/// [`FIRMWARE_IDLE_OFF_MS`] the driver keeps the GPU cores powered until it has been idle +/// this long ([`Hold`]). pub(crate) static IDLE_HOLD_MS: AtomicU32 = AtomicU32::new(FIRMWARE_IDLE_OFF_MS); +/// Use the same bounds as the existing idle-delay attribute. Missing DT policy +/// preserves the native delay; malformed or out-of-range values fail probe. +pub(super) fn idle_delay(requested: Option) -> kernel::error::Result { + let delay = requested.unwrap_or(FIRMWARE_IDLE_OFF_MS); + if !(FIRMWARE_IDLE_OFF_MS..=1000).contains(&delay) { + return Err(kernel::error::code::EINVAL); + } + Ok(delay) +} + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum IdleAction { None, @@ -237,6 +249,34 @@ pub(super) fn release(control: &mut C) -> kernel::error::Result { mod tests { use super::*; + #[test] + fn dt_idle_delay_is_bounded_and_absence_preserves_native_policy() { + assert_eq!(idle_delay(None), Ok(FIRMWARE_IDLE_OFF_MS)); + for delay in [2, 16, 1000] { + assert_eq!(idle_delay(Some(delay)), Ok(delay)); + } + for delay in [0, 1, 1001, u32::MAX] { + assert!(idle_delay(Some(delay)).is_err()); + } + } + + #[test] + fn frame_hold_survives_depth_one_gaps_but_releases_at_idle() { + let mut hold = Hold::new(); + assert_eq!(hold.step(0, true, 1, 16), HoldAction::Assert); + hold.asserted(); + for submission in 1..=8 { + // A completed depth-one dispatch followed by another short gap. + let now = submission * 3_000_000; + assert_eq!(hold.step(now, false, submission, 16), HoldAction::Recheck(16_000_000)); + assert!(hold.active()); + } + assert_eq!(hold.step(39_999_999, false, 8, 16), HoldAction::Recheck(1)); + assert_eq!(hold.step(40_000_000, false, 8, 16), HoldAction::Release); + hold.released(); + assert!(!hold.active()); + } + #[test] fn holds_until_idle_for_the_delay_then_releases() { let mut hold = Hold::new(); @@ -260,3 +300,43 @@ mod tests { assert_eq!(hold.step(1, false, 2, FIRMWARE_IDLE_OFF_MS), HoldAction::Release); } } + +unsafe extern "C" { + fn asahi_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) -> core::ffi::c_int; + fn asahi_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); + fn asahi_clpc_gpu_put(gpu: *mut core::ffi::c_void); +} + +/// DT-selected memory-demand provider. Its independent allocation outlives a +/// detached provider; a device link orders supplier removal and suspend. +pub(super) struct Demand(*mut core::ffi::c_void); + +// SAFETY: the provider's leaf spinlock serializes all access; no GPU callback. +unsafe impl Send for Demand {} +// SAFETY: shared access only changes that locked request, never its lifetime. +unsafe impl Sync for Demand {} + +impl Demand { + pub(super) fn new(dev: *mut kernel::bindings::device) -> kernel::error::Result { + let mut out = core::ptr::null_mut(); + // SAFETY: live platform device and writable result; C returns an owned reference. + let ret = unsafe { asahi_clpc_gpu_get(dev.cast(), &mut out) }; + if ret < 0 { + Err(kernel::error::Error::from_errno(ret)) + } else { + Ok(Self(out)) + } + } + + pub(super) fn set_active(&self, active: bool) { + // SAFETY: the retained reference is live; NULL denotes no DT provider. + unsafe { asahi_clpc_gpu_set_active(self.0, active) }; + } +} + +impl Drop for Demand { + fn drop(&mut self) { + // SAFETY: releases exactly the reference returned by get, clearing demand first. + unsafe { asahi_clpc_gpu_put(self.0) }; + } +} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 0297c619b4f93..c7d1761699a24 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -6,6 +6,7 @@ //! the device mutex, then drops that mutex before waiting for builders. A builder's lease is //! stored after its tentative mappings so those mappings are destroyed before the join completes. +use core::cell::Cell; use core::sync::atomic::{AtomicU64, Ordering}; use kernel::{ new_condvar, new_mutex, @@ -24,25 +25,37 @@ const EPOCH: u64 = CLOSED << 3; /// packet cancellation, logical close, VM status and device loss even when no /// capacity/recovery event arrives. Wakes and nested constructors never renew it. pub(super) struct Admission<'a> { - deadline: u64, + deadline: Cell, + started: u64, check: &'a dyn Fn() -> Result, } impl<'a> Admission<'a> { pub(super) fn new(deadline: u64, check: &'a dyn Fn() -> Result) -> Self { - Self { deadline, check } + Self { deadline: Cell::new(deadline), started: super::now_ns(), check } + } + + /// A recovery is pending or this owner's spared work drains: wait as long as the + /// watchdog may take to settle it (RECOVERY_WAIT_MS from the start) instead of failing. + pub(super) fn extend_for_recovery(&self) { + let limit = self + .started + .saturating_add(u64::from(super::timeout::RECOVERY_WAIT_MS) * 1_000_000); + if self.deadline.get() < limit { + self.deadline.set(limit); + } } pub(super) fn check(&self) -> Result { (self.check)()?; - if super::now_ns() >= self.deadline { return Err(EBUSY); } + if super::now_ns() >= self.deadline.get() { return Err(EBUSY); } Ok(()) } /// Ignore condvar remaining-time reports: only the monotonic deadline counts. pub(super) fn slice_ms(&self) -> Result { self.check()?; - let remaining = self.deadline.saturating_sub(super::now_ns()); + let remaining = self.deadline.get().saturating_sub(super::now_ns()); if remaining == 0 { return Err(EBUSY); } Ok(remaining.div_ceil(1_000_000).min(10) as u32) } @@ -55,20 +68,37 @@ pub(super) struct Gate { wait: Mutex<()>, #[pin] changed: CondVar, + /// Scheduler admission waiting for publication to reopen. + available: Option>, } impl Gate { + #[cfg(CONFIG_KUNIT = "y")] pub(super) fn new() -> Result> { + Self::with_waiters(None) + } + + /// A gate whose reopening, failure and removal also wake `available`. + pub(super) fn with_waiters( + available: Option>, + ) -> Result> { Arc::pin_init( pin_init!(Self { state: AtomicU64::new(EPOCH), wait <- new_mutex!((), "g17::Gate::wait"), changed <- new_condvar!("g17::Gate::changed"), + available, }), GFP_KERNEL, ) } + fn wake_waiters(&self) { + if let Some(available) = &self.available { + available.notify(); + } + } + pub(super) fn try_enter(self: &Arc) -> Result { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -120,19 +150,20 @@ impl Gate { } } - /// Packet admission may wait for recovery only within its original budget. + /// Only a recovery closes the gate: packet admission waits for it up to + /// RECOVERY_WAIT_MS from the admission's start (Admission::extend_for_recovery). /// Called with no device mutex or existing preparation lease. pub(super) fn enter_until(self: &Arc, admission: &Admission<'_>) -> Result { admission.check()?; match self.try_enter() { - Err(error) if error == EAGAIN => {} + Err(error) if error == EAGAIN => admission.extend_for_recovery(), result => return result, } let mut wait = self.wait.lock(); loop { admission.check()?; match self.try_enter() { - Err(error) if error == EAGAIN => {} + Err(error) if error == EAGAIN => admission.extend_for_recovery(), result => return result, } let _ = self.changed.wait_timeout( @@ -162,20 +193,55 @@ impl Gate { } } + /// Also wakes publishers in wait_closure(); they then block in enter(). pub(super) fn close(&self) { self.invalidate(0); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + /// Opaque snapshot for wait_closure(), taken where recovery was observed. + pub(super) fn snapshot(&self) -> u64 { + self.state.load(Ordering::Acquire) + } + + /// A publisher that observed recovery (host pending, or the firmware's state + /// ahead of the event worker, which always follows with a restart and the + /// closure) sleeps until this gate has closed since `seen`. enter() then + /// blocks until recovery reopens it. The deadline is only a failure bound. + pub(super) fn wait_closure(&self, seen: u64, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + loop { + let state = self.state.load(Ordering::Acquire); + if state & (REMOVED | FAILED) != 0 { + return Err(ENODEV); + } + if state & CLOSED != 0 || state & !COUNT_MASK != seen & !COUNT_MASK { + return Ok(()); + } + let remaining = deadline.saturating_sub(super::now_ns()); + if remaining == 0 { + return Err(ETIMEDOUT); + } + let ms = u32::try_from(remaining.div_ceil(1_000_000)).unwrap_or(u32::MAX); + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(ms)); + } } pub(super) fn remove(&self) { self.invalidate(REMOVED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } pub(super) fn fail(&self) { self.invalidate(FAILED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } /// Runtime failure/recovery must not join an indefinitely stalled builder. @@ -199,7 +265,7 @@ impl Gate { /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. pub(super) fn reopen(&self) -> bool { - let _wait = self.wait.lock(); + let wait = self.wait.lock(); let mut previous = self.state.load(Ordering::Acquire); loop { if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { @@ -213,6 +279,8 @@ impl Gate { ) { Ok(_) => { self.changed.notify_all(); + drop(wait); + self.wake_waiters(); return true; } Err(current) => previous = current, @@ -268,6 +336,20 @@ mod growth_gate_tests { mod compute_deadline_tests { use super::*; + #[test] + fn a_recovery_extends_the_deadline_once_from_the_start() { + let check = || Ok(()); + let admission = Admission::new(super::super::now_ns(), &check); + assert_eq!(admission.check(), Err(EBUSY)); + admission.extend_for_recovery(); + assert_eq!(admission.check(), Ok(())); + let limit = admission.started + + u64::from(super::super::timeout::RECOVERY_WAIT_MS) * 1_000_000; + assert_eq!(admission.deadline.get(), limit); + admission.extend_for_recovery(); + assert_eq!(admission.deadline.get(), limit); + } + #[test] fn spent_budget_refuses_even_an_open_gate() { let gate = Gate::new().unwrap(); @@ -278,13 +360,16 @@ mod compute_deadline_tests { } #[test] - fn closed_gate_expires_without_any_recovery_wake() { + fn closed_gate_waits_past_the_budget_for_recovery() { let gate = Gate::new().unwrap(); gate.close(); let start = super::super::now_ns(); - let check = || Ok(()); + // Cancel well after the 20 ms budget: only the recovery extension keeps it waiting. + let check = || { + if super::super::now_ns() - start >= 60_000_000 { Err(ECANCELED) } else { Ok(()) } + }; let admission = Admission::new(start + 20_000_000, &check); - assert!(matches!(gate.enter_until(&admission), Err(error) if error == EBUSY)); + assert!(matches!(gate.enter_until(&admission), Err(error) if error == ECANCELED)); assert!(super::super::now_ns() - start < 500_000_000); gate.wait_drained(); assert!(gate.reopen()); @@ -307,9 +392,9 @@ mod compute_deadline_tests { #[test] fn recovery_and_binding_retry_churn_cannot_renew_budget() { let gate = Gate::new().unwrap(); - let start = super::super::now_ns(); + let deadline = super::super::now_ns() + 20_000_000; let check = || Ok(()); - let admission = Admission::new(start + 20_000_000, &check); + let admission = Admission::new(deadline, &check); loop { // Re-entry models a channel taken between admission/publication. // Recovery joins each dropped lease, then wakes the same budget. @@ -317,12 +402,17 @@ mod compute_deadline_tests { gate.changed.notify_all(); gate.wait_drained(); assert!(gate.reopen()); + // However often the gate reopened, an attempt that starts at the deadline + // is refused. Timing the whole loop would also count time descheduled. + let attempted = super::super::now_ns(); match gate.enter_until(&admission) { - Ok(lease) => drop(lease), + Ok(lease) => { + assert!(attempted < deadline); + drop(lease); + } Err(error) => { assert_eq!(error, EBUSY); break; } } } - assert!(super::super::now_ns() - start < 500_000_000); assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); } diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index ebdc79d2b2dd3..f09e629aa047e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -9,6 +9,7 @@ //! a graph or waiting for GPU work. Capacity waiters sleep on the release event. use crate::g17::{ + fw::kick::KickDependency, hazard::{self, Footprint, Plan}, job::Packet, }; @@ -43,6 +44,9 @@ struct InFlight { slot: Option, ticket: u64, footprint: Arc, + /// Fragment kick of the published render; a later same-VM overlap of + /// another pair waits for it in firmware. + fragment: Option, } const QUARANTINE_WAIT_MS: u32 = 1000; @@ -67,11 +71,19 @@ struct State { /// their own fence with EBUSY instead of a device-loss error, so the VM stays usable. spared: u64, kill_hold: u64, + /// Slots whose kill hold ended at their killed context's consumed scheduler release, + /// and when. A new owner takes such a pair last, the one released longest ago first. + killed: u64, + killed_at: [u64; cfg::RENDER_SLOTS], + /// The physical queue table still has room for a pair built in an unowned slot. The + /// runtime updates this whenever the table changes. + build_room: bool, released: u64, epoch: u64, footprints: KVec, next_ticket: u64, } + impl State { const MASK: u64 = (u64::MAX >> (64 - cfg::RENDER_SLOTS)) & !1; const fn new() -> Self { @@ -83,6 +95,9 @@ impl State { quarantined: 0, spared: 0, kill_hold: 0, + killed: 0, + killed_at: [0; cfg::RENDER_SLOTS], + build_room: true, released: 0, epoch: 0, footprints: KVec::new(), @@ -116,6 +131,15 @@ impl State { self.epochs[slot] = self.epoch; } fn take_idle(&mut self, slot: usize, owner: u64) -> u8 { + if self.owners[slot] != owner { + crate::g17::trace::teardown( + 6, + slot as u64, + self.owners[slot], + owner | u64::from(self.killed & (1u64 << slot) != 0) << 63, + ); + self.killed &= !(1u64 << slot); + } self.refs[slot] += 1; self.active |= 1u64 << slot; self.released &= !(1u64 << slot); @@ -161,13 +185,23 @@ impl State { } let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; let eligible = |slot: usize| free & (1u64 << slot) != 0; - // Reuse a closed, exactly retired owner before consuming a new graph. + let released = |slot: usize| self.released & (1u64 << slot) != 0; + let unowned = || (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0); + // Reuse a closed, exactly retired owner before consuming a new graph. A killed + // context's pair comes after a new graph while one can still be built: on Neo, new + // owners of such pairs had renders that executed but never stored their stamps. + // Once the queue table is full it is reused, the one released longest ago first, + // so killed pairs never strand physical queues. let slot = (0..cfg::RENDER_SLOTS) - .filter(|slot| eligible(*slot) && self.released & (1u64 << slot) != 0) + .filter(|slot| eligible(*slot) && released(*slot) && self.killed & (1u64 << slot) == 0) .min_by_key(|slot| self.epochs[*slot]) + .or_else(|| if self.build_room { unowned() } else { None }) .or_else(|| { - (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) - })?; + (0..cfg::RENDER_SLOTS) + .filter(|slot| eligible(*slot) && released(*slot)) + .min_by_key(|slot| self.killed_at[*slot]) + }) + .or_else(unowned)?; let first = self.take_idle(slot, owner); self.claim_sibling(owner); Some(first) @@ -176,7 +210,8 @@ impl State { if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { return; } - let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + // A speculative sibling never takes a killed context's pair. + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold & !self.killed; let eligible = |slot: usize| free & (1u64 << slot) != 0; let sibling = (0..cfg::RENDER_SLOTS) .filter(|slot| { @@ -186,6 +221,10 @@ impl State { }) .min_by_key(|slot| self.epochs[*slot]) .or_else(|| { + // Nor an unowned slot once no pair can be built there. + if !self.build_room { + return None; + } (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) }); if let Some(slot) = sibling { @@ -219,6 +258,7 @@ impl State { let bit = 1u64 << slot; let old = self.owners[slot]; if old == 0 + || self.killed & bit != 0 || owner == Some(old) || expected.is_some_and(|expected| expected != old) || self.refs[slot] != 0 @@ -294,6 +334,28 @@ impl State { ) } + /// Whether `pick` would return a slot, without taking it. + fn can_pick(&self, owner: u64, plan: Plan) -> bool { + let usable = |slot: usize| { + self.owners[slot] == owner + && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 + && self.refs[slot] < cfg::RENDER_DEPTH + }; + match plan { + Plan::Wait => false, + Plan::Pair(slot) => usize::from(slot) < cfg::RENDER_SLOTS && usable(usize::from(slot)), + Plan::Any if owner == 0 => false, + Plan::Any if self.owners.contains(&owner) => (0..cfg::RENDER_SLOTS).any(usable), + Plan::Any => { + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + (0..cfg::RENDER_SLOTS).any(|slot| { + free & (1u64 << slot) != 0 + && (self.released & (1u64 << slot) != 0 || self.owners[slot] == 0) + }) + } + } + } + fn pick(&mut self, owner: u64, plan: Plan) -> Option { match plan { Plan::Any => self.acquire(owner), @@ -332,6 +394,7 @@ impl State { slot, ticket, footprint: request.footprint.clone(), + fragment: None, }, GFP_KERNEL, ) @@ -339,6 +402,12 @@ impl State { Some((owner, request.sequence)) } + fn position(&self, slot: u8, (owner, sequence): (u64, u64)) -> Option { + self.footprints.iter().position(|entry| { + entry.slot == Some(slot) && entry.owner == owner && entry.sequence == sequence + }) + } + fn forget_waiter(&mut self, ticket: Option) { if let Some(ticket) = ticket { self.footprints @@ -366,9 +435,22 @@ pub(crate) struct Pool { queues: AtomicU32, /// Logical queues admitted at most: each must be able to hold a render pair. max_queues: u32, + /// Logical-queue creation waiting for a returned slot. + waiters: Option>, + /// Render publication waiting for render capacity: woken with every + /// release event above. + renders: Option>, } impl Pool { + #[cfg(CONFIG_KUNIT = "y")] pub(crate) fn new(max_queues: u32) -> Result> { + Self::with_waiters(max_queues, None, None) + } + pub(crate) fn with_waiters( + max_queues: u32, + waiters: Option>, + renders: Option>, + ) -> Result> { if max_queues > MAX_EXECUTION_CONTEXTS { return Err(EINVAL); } @@ -378,10 +460,19 @@ impl Pool { released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), max_queues, + waiters, + renders, }), GFP_KERNEL, ) } + /// Every pair or lease return, owner close and quarantine change. + fn returned(&self) { + self.released.notify_all(); + if let Some(renders) = &self.renders { + renders.notify(); + } + } /// Logical queues currently counted, including one being created. pub(crate) fn queues(&self) -> u32 { self.queues.load(Ordering::Acquire) @@ -403,6 +494,16 @@ impl Pool { } } } + /// The owner's pair is quarantined behind another VM's recovery (spared) and drains. + pub(crate) fn spared_quarantined(&self, owner: u64) -> bool { + let state = self.state.lock(); + state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & state.spared & (1u64 << slot) != 0) + } + pub(crate) fn acquire_render( self: &Arc, owner: u64, @@ -434,17 +535,22 @@ impl Pool { .position(|id| *id == owner) .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) { - if now >= quarantine_deadline { - // Behind an innocent pair's quarantine (another VM wedged the GPU) only this - // publication fails, not the VM: it recovers once the pair drains. - let spared = state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.spared & (1u64 << slot) != 0); - return Err(if spared { EBUSY } else { EIO }); + // Behind an innocent pair's quarantine (another VM wedged the GPU) wait for the + // pair to drain until the caller's deadline, which the caller extends while it + // drains (RECOVERY_WAIT_MS); a guilty pair's publication fails after a second. + let spared = state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.spared & (1u64 << slot) != 0); + if now >= quarantine_deadline && !spared { + return Err(EIO); } - let slice = remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?); + let slice = if spared { + remaining_ms + } else { + remaining_ms.min(wait_slice_ms(quarantine_deadline, now)?) + }; if matches!(self.released.wait_interruptible_timeout( &mut state, kernel::time::msecs_to_jiffies(slice) ), CondVarTimeoutResult::Signal { .. }) { @@ -522,6 +628,24 @@ impl Pool { record, })) } + /// Whether `acquire_render` would admit this request now, without + /// admitting it: the scheduler's prepare step waits for this. + pub(crate) fn admits_render(&self, owner: u64, request: Option>) -> bool { + let state = self.state.lock(); + if owner == 0 || state.refusal(owner).is_err() { + return false; + } + if state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) + { + return false; + } + let plan = state.plan(owner, request, None); + state.can_pick(owner, plan) + } pub(crate) fn warmable_sibling( self: &Arc, owner: u64, @@ -558,7 +682,7 @@ impl Pool { } } drop(state); - self.released.notify_all(); + self.returned(); } /// Active leases survive this transition. Their later drop is still the /// event which releases capacity, so a stopped owner is never double-freed. @@ -575,9 +699,27 @@ impl Pool { state.quarantined &= !released; state.spared &= !released; state.kill_hold = 0; + state.killed = 0; drop(state); - self.released.notify_all(); + self.returned(); + } + + /// Whether the physical queue table can hold another pair (see `State::build_room`). + pub(crate) fn note_build_room(&self, room: bool) { + let mut state = self.state.lock(); + if state.build_room != room { + state.build_room = room; + drop(state); + self.returned(); + } } + + /// A killed context's pair is free for a new owner. + pub(crate) fn killed_pair_free(&self) -> bool { + let state = self.state.lock(); + state.killed & state.released & !state.active & !state.quarantined & !state.kill_hold & State::MASK != 0 + } + } #[kunit_tests(asahi_g17_admission_accounting)] @@ -653,9 +795,18 @@ mod accounting_tests { pub(crate) struct QueueSlot { pool: Arc, } +impl QueueSlot { + /// The device's admission pool, which lives as long as the firmware session. + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } +} impl Drop for QueueSlot { fn drop(&mut self) { self.pool.queues.fetch_sub(1, Ordering::AcqRel); + if let Some(waiters) = &self.pool.waiters { + waiters.notify(); + } } } @@ -691,7 +842,7 @@ impl RenderLease { } state.quarantined |= bit; drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. @@ -705,24 +856,83 @@ impl RenderLease { self.rehabilitate_after_processor_stop(); } } +impl RenderLease { + /// Called under the device mutex once the render's kicks are committed, + /// including a replay, which supersedes the earlier kick. + pub(crate) fn note_publication(&self, fragment: KickDependency) { + let Some(record) = self.record else { + return; + }; + let mut state = self.pool.state.lock(); + if let Some(index) = state.position(self.slot, record) { + state.footprints[index].fragment = Some(fragment); + } + } +} + +impl Pool { + /// Under the device mutex, which orders every publication: the newest + /// published fragment kick of each other pair whose live same-VM render + /// overlaps `footprint`. Quarantined pairs are skipped as in admission. + pub(crate) fn publication_barriers( + &self, + owner: u64, + slot: u8, + footprint: &Footprint, + barriers: &mut [KickDependency; hazard::PUBLICATION_BARRIERS], + ) -> Result { + let state = self.state.lock(); + let mut slots = [0u8; hazard::PUBLICATION_BARRIERS]; + let mut count = 0; + for entry in state.footprints.iter() { + let (Some(other), Some(fragment)) = (entry.slot, entry.fragment) else { + continue; + }; + if other == slot + || entry.owner == owner + || state.quarantined & (1u64 << other) != 0 + || !footprint.same_vm(&entry.footprint) + || !footprint.overlaps(&entry.footprint) + { + continue; + } + match slots[..count].iter().position(|known| *known == other) { + // One fragment ring completes in kick order; same QID, so the + // word orders by timestamp. + Some(index) => { + if fragment.word() > barriers[index].word() { + barriers[index] = fragment; + } + } + // Admission bounds the pairs; see hazard::plan. + None if count == hazard::PUBLICATION_BARRIERS => return Err(EBUSY), + None => { + slots[count] = other; + barriers[count] = fragment; + count += 1; + } + } + } + Ok(count) + } +} + impl Drop for RenderLease { fn drop(&mut self) { let mut state = self.pool.state.lock(); - if let Some((owner, sequence)) = self.record { - if let Some(index) = state.footprints.iter().position(|entry| { - entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence - }) { + if let Some(record) = self.record { + if let Some(index) = state.position(self.slot, record) { state.footprints.swap_remove(index); } } state.release(self.slot); drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } } impl State { - fn hold_for_exit_kill(&mut self, owner: u64, slots: u64) -> bool { + fn hold_for_exit_kill(&mut self, owner: u64, slots: u64, executed: u64) -> bool { if owner == 0 || slots & !Self::MASK != 0 { return false; } @@ -731,10 +941,10 @@ impl State { if slots & bit == 0 { continue; } + // A pair whose commands all ran is held with them (see `Pair::executed_for_kill`). if self.owners[slot] != owner - || self.refs[slot] != 0 - || self.active & bit != 0 - || self.released & bit == 0 + || (executed & bit == 0 + && (self.refs[slot] != 0 || self.active & bit != 0 || self.released & bit == 0)) { return false; } @@ -745,18 +955,39 @@ impl State { } impl Pool { /// Hold the departed owner's idle pairs before publishing its exit record. - /// Successful publication retains these bits until both processors stop. - pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64) -> bool { - self.state.lock().hold_for_exit_kill(owner, slots) + /// Successful publication retains these bits until the owner's scheduler release is + /// consumed (`release_killed_slots` at detach), or until both processors stop. + pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64, executed: u64) -> bool { + self.state.lock().hold_for_exit_kill(owner, slots, executed) } - /// Undo only a hold whose control record was not published. + /// Undo a hold whose control record was not published. pub(crate) fn release_exit_kill_hold(&self, slots: u64) { if slots == 0 { return; } self.state.lock().kill_hold &= !slots; - self.released.notify_all(); + self.returned(); + } + + /// End the kill hold of `slots` once their killed context's scheduler release was + /// consumed (the firmware no longer names these pairs). A new owner takes them last. + pub(crate) fn release_killed_slots(&self, slots: u64) { + let mut state = self.state.lock(); + let held = state.kill_hold & slots & State::MASK; + if held == 0 { + return; + } + let now = crate::g17::now_ns(); + state.kill_hold &= !held; + state.killed |= held; + for slot in 0..cfg::RENDER_SLOTS { + if held & (1u64 << slot) != 0 { + state.killed_at[slot] = now; + } + } + drop(state); + self.returned(); } } @@ -785,10 +1016,13 @@ mod fault_capacity_tests { let queue = pool.reserve_queue().unwrap(); let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); let bit = 1u64 << lease.slot(); + assert!(!pool.spared_quarantined(1)); lease.quarantine(true); assert_ne!(pool.state.lock().spared & bit, 0); + assert!(pool.spared_quarantined(1)); lease.quarantine(false); assert_eq!(pool.state.lock().spared & bit, 0); + assert!(!pool.spared_quarantined(1)); // Once guilty, a later spared failure does not make it spared again. lease.quarantine(true); assert_eq!(pool.state.lock().spared & bit, 0); @@ -831,13 +1065,86 @@ mod fault_capacity_tests { let bit = 1u64 << lease.slot(); pool.release_owner(1); drop(lease); - assert!(pool.hold_render_slots_for_exit_kill(1, bit)); + assert!(pool.hold_render_slots_for_exit_kill(1, bit, 0)); assert!(!pool.reassign_unused_spare(bit.trailing_zeros() as u8, Some(1), Some(2))); // A failed (unpublished) control may roll back its hold. Published kills // keep the existing processor-stop rule; this test must not weaken it. pool.release_retained_owners_after_processor_stop(); assert_eq!(pool.state.lock().kill_hold, 0); } + + #[test] + fn consumed_scheduler_release_ends_the_kill_hold() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + let slot = lease.slot(); + let bit = 1u64 << slot; + pool.release_owner(1); + drop(lease); + assert!(pool.hold_render_slots_for_exit_kill(1, bit, 0)); + assert!(!pool.reassign_unused_spare(slot, Some(1), Some(2))); + pool.release_killed_slots(bit); + assert_eq!(pool.state.lock().kill_hold, 0); + assert_eq!(pool.state.lock().killed, bit); + assert!(pool.killed_pair_free()); + // The killed context's pair is never reassigned as a spare, and a new owner takes + // a new graph before it. + assert!(!pool.reassign_unused_spare(slot, Some(1), Some(2))); + let next = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + assert_ne!(next.slot(), slot); + drop(next); + pool.release_retained_owners_after_processor_stop(); + assert_eq!(pool.state.lock().killed, 0); + } + + #[test] + fn killed_pairs_are_taken_last_oldest_first() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + for slot in 1..cfg::RENDER_SLOTS { + state.owners[slot] = 100 + slot as u64; + state.released |= 1u64 << slot; + state.killed |= 1u64 << slot; + state.killed_at[slot] = 1000 - slot as u64; + } + // One pair released without a kill. + state.killed &= !(1u64 << 7); + } + let first = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + assert_eq!(first.slot(), 7); + let second = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + assert_eq!(usize::from(second.slot()), cfg::RENDER_SLOTS - 1); + assert_eq!(pool.state.lock().killed & (1u64 << second.slot()), 0); + } + + #[test] + fn a_full_queue_table_reuses_killed_pairs_before_unowned_slots() { + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); + { + let mut state = pool.state.lock(); + for slot in 1..8 { + state.owners[slot] = 100 + slot as u64; + state.released |= 1u64 << slot; + state.killed |= 1u64 << slot; + state.killed_at[slot] = 1000 - slot as u64; + } + } + // With room in the queue table a new owner builds a pair in an unowned slot. + let built = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); + assert!(usize::from(built.slot()) >= 8); + // Without room it takes the killed pair released longest ago. + pool.note_build_room(false); + let reused = pool.acquire_render(2, None, u64::MAX, || Ok(())).unwrap(); + assert_eq!(reused.slot(), 7); + assert_eq!(pool.state.lock().killed & (1u64 << 7), 0); + // It claims no sibling: neither another killed pair nor an unowned slot. + let state = pool.state.lock(); + for slot in 1..7 { + assert_eq!(state.owners[slot], 100 + slot as u64); + } + assert_eq!(state.owners.iter().filter(|owner| **owner == 2).count(), 1); + } } impl Pool { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 713809907dd4b..55994066825f8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -15,7 +15,7 @@ use crate::g17::{ queue::*, variant::PointerLayout, }, - object::{Allocator, CpuMap, KernelObject}, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; use crate::{hw::t8140::dynamic, mmu}; use core::sync::atomic::{fence, Ordering}; @@ -30,10 +30,21 @@ const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; +const NATIVE_WAIT_OFFSET: usize = 0x900; +const NATIVE_WAIT_STRIDE: usize = 0x60; +// A submission can name every physical compute channel. Keep the complete +// wait list inside its own cached slot, including the last retained slot. +const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET + + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE) + .next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; const POINTERS: usize = 0x100; +/// Producer/consumer words live on their own uncached page after the graph, +/// which the firmware reads through a cached mapping (queue record, job-list +/// head and item ring), as native does. +const CURSORS: usize = GRAPH_SIZE; const ITEMS: usize = 0x4000; const STAMP: usize = 0x300; @@ -83,24 +94,32 @@ pub(crate) struct Binding { /// Descriptor ring of one compute QID with its kernel-VM alias. Built without /// the device mutex; installed into a vacant graph before an owner binds. pub(crate) struct DescriptorRing { - object: KernelObject, - high: mmu::KernelMapping, + // Drop the GPU alias before the retained storage returns to its pool. + low: mmu::KernelMapping, + object: PooledObject, } impl DescriptorRing { - pub(crate) fn new(alloc: &Allocator<'_>) -> Result { - let page = mmu::UAT_PGSZ as u64; - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let object = alloc.lower( - DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - page, - prot, + /// Firmware reads every descriptor through a cached kernel mapping, as it + /// does for render. The pool keeps that mapping until processor stop; the + /// hardware reads the same pages through an uncached canonical low alias. + pub(crate) fn new(alloc: &Allocator<'_>, objects: &Arc) -> Result { + let size = DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; + let object = objects.allocate( + alloc, + Family::Descriptor, + size, + size, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let mut high = alloc.uat.geometry().kernel_range(); - high.start += dynamic::KERNEL_OFFSET; - let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; - Ok(Self { object, high }) + let low = object.alias_in( + alloc.uat.kernel_lower_vm(), + dynamic::LOWER, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + )?; + Ok(Self { low, object }) } } @@ -112,10 +131,14 @@ struct Graph { usage: KernelObject, operand: KernelObject, support: KernelObject, - graph: KernelObject, + graph: PooledObject, context_low: KernelObject, context_high: KernelObject, + /// Announcements, which stay uncached for the firmware. records: KernelObject, + /// Configuration, predecessor and completion-wait records at the same + /// offsets, which the firmware reads through a cached mapping. + cached_records: PooledObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, /// Host words of the pointer block in this firmware build. @@ -127,6 +150,7 @@ struct Graph { impl Graph { fn new( alloc: &Allocator<'_>, + objects: &Arc, context: &Context, pointers: PointerLayout, operand_state: &'static OperandState, @@ -134,7 +158,7 @@ impl Graph { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let ring = DescriptorRing::new(alloc)?; + let ring = DescriptorRing::new(alloc, objects)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(operand_state.size, page, prot, cpu)?; @@ -144,11 +168,18 @@ impl Graph { support.write(0x100, support_va + page + 4)?; support.write(0x110, 0x50u32)?; support.write(mmu::UAT_PGSZ + 4, 1u32)?; - let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; + let mut graph = objects.allocate( + alloc, + Family::QueueState, + GRAPH_SIZE + mmu::UAT_PGSZ, + GRAPH_SIZE, + prot, + cpu, + )?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + POINTERS as u64, + pointers_va: graph_va + CURSORS as u64, items_va: graph_va + ITEMS as u64, job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), @@ -159,9 +190,21 @@ impl Graph { view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; + graph.initialize::(CURSORS, |view| { + *view = PointerBlock::new(&pointers)?; + Ok(()) + })?; let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + let cached_records = objects.allocate( + alloc, + Family::QueueConfig, + RECORD_SLOTS * CACHED_RECORD_STRIDE, + RECORD_SLOTS * CACHED_RECORD_STRIDE, + prot, + cpu, + )?; Ok(Self { ring: Some(ring), preempt, @@ -172,6 +215,7 @@ impl Graph { context_low, context_high, records, + cached_records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, pointers, @@ -237,9 +281,12 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + self.pointers.producer)?.load(Ordering::Relaxed), - consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), - count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), + producer: self + .graph + .word(CURSORS + self.pointers.producer)? + .load(Ordering::Relaxed), + consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), + count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), }) } fn stamp(&self) -> Result { @@ -250,12 +297,12 @@ impl Graph { fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; - Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) + Ok((ring.object.gpu_va() + offset, ring.low.iova() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { let ring = self.ring.as_ref().ok_or(EAGAIN)?; ring.object - .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) + .map_alias(vm, ring.low.iova(), mmu::PROT_GPU_SHARED_RO) } /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed @@ -289,11 +336,13 @@ impl Graph { return Err(EIO); } for index in 0..ITEM_RING_ENTRIES as usize { - self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); + self.graph + .dword(ITEMS + index * 8)? + .store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); self.graph - .word(POINTERS + self.pointers.producer)? + .word(CURSORS + self.pointers.producer)? .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); @@ -519,12 +568,24 @@ pub(crate) struct Queue { replay_error: Option, replay_prepared: bool, exit: Option, + /// Since when the kill of this failed queue's context waits for other contexts' compute, + /// and whether that wait was reported. + kill_wait: Option<(u64, bool)>, installed: bool, /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, + /// Last configuration record the firmware consumed, and its recovery + /// epoch. A command repeats the record only when it differs. + config_cache: Option<(QueueConfig, u64)>, ordinal: u32, submitted: u32, + /// The stamp of the newest command the firmware reported failed ([`Self::note_fault`]), + /// while it is newer than the stored stamp. + fault_stamp: Option, + /// Prepared packets of the bound owner about to publish (see [`Self::claim`]). + claims: u32, + last_published: u64, last_end: u64, released: bool, retirement_ready: bool, @@ -538,6 +599,8 @@ pub(crate) struct Queue { quarantine_error: Option, failure_status_pending: bool, retirement_proved: bool, + source_readers: Arc, + source_hint: crate::g17::sync_input::ReaderHint, } impl Queue { @@ -545,6 +608,7 @@ impl Queue { /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( alloc: &Allocator<'_>, + objects: &Arc, backing: Backing, owner: u64, context: Arc, @@ -552,11 +616,13 @@ impl Queue { operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context, pointers, operand_state)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; context.mark_published(); + context.note_compute_binding(true); + crate::g17::trace::bind(crate::g17::trace::COMPUTE, kick.id().qid(), context.id(), 0); Ok(Self { owner: Some(owner), retained_owner: None, @@ -579,11 +645,16 @@ impl Queue { replay_error: None, replay_prepared: false, exit: None, + kill_wait: None, installed: false, outer_started: 0, published_config: None, + config_cache: None, ordinal: 0, submitted: 0, + fault_stamp: None, + claims: 0, + last_published: 0, last_end: 0, released: false, retirement_ready: false, @@ -597,8 +668,44 @@ impl Queue { quarantine_error: None, failure_status_pending: false, retirement_proved: false, + source_readers: Arc::new(core::sync::atomic::AtomicU32::new(0), GFP_KERNEL)?, + source_hint: crate::g17::sync_input::ReaderHint::new(), + }) + } + /// The firmware failed the command whose completion stamp is `stamp`. It writes a + /// zero-duration completion record for it, which quarantines the queue, but stores no + /// stamp: without this the quarantine would wait for its retirement witness forever and + /// keep the queue's backing. Commands published after the failed one still need their own + /// stamp or record. + pub(crate) fn note_fault(&mut self, stamp: u32) { + if self + .fault_stamp + .is_none_or(|old| retirement::stamp_newer(stamp, old)) + { + self.fault_stamp = Some(stamp); + } + } + + /// The firmware failed a command of this quarantined queue, and the queue still waits for + /// its retirement witness: commands published after the failed one, which the firmware + /// never runs, or the failed command itself, whose ring entry it does not consume (a ray + /// traversal's, on Neo). Only a kill of the queue's context gives them back. + pub(crate) fn fault_pending(&self) -> bool { + self.fault_stamp.is_some() + && self.quarantined + && self.awaiting_witness + && self.exit.is_none() + } + + /// The stamp that proves retirement: the stored one, or a newer reported fault's. + fn witness_stamp(&self) -> Result { + let stored = self.graph.stamp()?; + Ok(match self.fault_stamp { + Some(fault) if retirement::stamp_newer(fault, stored) => fault, + _ => stored, }) } + pub(crate) fn qid(&self) -> u8 { self.kick.id().qid() } @@ -626,12 +733,24 @@ impl Queue { } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + !self + .context() + .is_some_and(|bound| Arc::ptr_eq(bound, context)) || (self.active.len == 0 && self.replays.len == 0) } pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined } + /// The oldest command is a published long job (ray traversal or AS build) whose + /// completion record the driver has not seen yet. + pub(crate) fn long_job_running(&self) -> bool { + !self.quarantined + && self.active.front().is_some_and(|active| { + active.observed.is_none() + && active.packet.is_published() + && active.packet.command.long_job() + }) + } /// Quarantines without a pending witness no longer describe live pipe order. pub(crate) fn timeout_head(&self) -> Option { if self.quarantined && !self.awaiting_witness { @@ -659,6 +778,27 @@ impl Queue { visit(active.packet.completion.status()); } } + /// A recovery record names this queue and the completion stamp of the + /// command it interrupted, `kick << 8` like the queue's stamp. Without a + /// stamp match the oldest published command is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self.active.iter().find(|active| { + active + .ticket + .is_some_and(|ticket| (ticket.kick as u32) & 0xff_ffff == sequence) + }) { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.ticket.is_some()) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { for active in self.active.iter() { active @@ -672,7 +812,10 @@ impl Queue { qid: self.qid(), // Accounting completes only for bound owners; a detached queue has no // pending work and never reaches this. - qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), + qos: self + .binding + .as_ref() + .map_or(0, |binding| binding.context.qos_id()), data_master: DataMaster::Compute as u8, } } @@ -689,7 +832,8 @@ impl Queue { } pub(crate) fn pool_release_stalled(&self, now: u64) -> bool { - self.pool_release_pending() && crate::g17::timeout::background_stalled(self.release_sent_ns, now) + self.pool_release_pending() + && crate::g17::timeout::background_stalled(self.release_sent_ns, now) } /// A closed owner can still retain its last commands or their completion @@ -698,7 +842,12 @@ impl Queue { pub(crate) fn release_settling(&self) -> bool { // release_owner keeps the old owner while its last commands retire. // Waiting only once owner is None misses precisely that interval. - release_pending(self.released, self.quarantined, self.active.len, self.retire_pending) + release_pending( + self.released, + self.quarantined, + self.active.len, + self.retire_pending, + ) } fn selected(&self, masks: Option<[u64; 2]>) -> bool { @@ -706,33 +855,32 @@ impl Queue { || self.retire_pending || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) } - /// Stage every unlatched exact identity before the device's single shared-ring scan. - pub(crate) fn request_observations( - &self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + /// Stage every unlatched exact identity before the device's single shared-ring scan, + /// whichever queues the scan's batch retires. + pub(crate) fn request_observations(&self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(false); } + batch.stage_frontier(self.qid(), self.kick.timestamp().get())?; let mut requested = false; + let mut stamp = None; for active in self.active.iter() { let Some(ticket) = active.ticket else { break; }; if active.observed.is_none() { - batch.request(&ticket)?; + let stamp = match stamp { + Some(stamp) => stamp, + None => *stamp.insert(self.graph.stamp()?), + }; + batch.request(&ticket, retirement::stamp_covers(stamp, ticket.kick))?; requested = true; } } Ok(requested) } - pub(crate) fn latch_observations( - &mut self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + pub(crate) fn latch_observations(&mut self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(()); } for active in self.active.iter_mut() { @@ -836,6 +984,7 @@ impl Queue { self.retire_pending = false; let mut oldest = true; while let Some(front) = self.active.front() { + let mut restart = None; let result = (|| { let ticket = front.ticket.ok_or(EIO)?; self.valid_ticket(front, &ticket)?; @@ -855,7 +1004,18 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = retirement::record_aged(front.observed_at, now_ns()); + let record_aged = !retirement::stamp_covers(stamp, ticket.kick) + && match retirement::stamp_wait(front.observed_at, now_ns(), || { + // A node that cannot be read keeps the bound from the record. + front.work.drained().unwrap_or(true) + }) { + retirement::StampWait::Waiting => false, + retirement::StampWait::Restart(now) => { + restart = Some(now); + false + } + retirement::StampWait::Aged => true, + }; let timestamps = retirement::poll( &ticket, front.observed, @@ -887,6 +1047,16 @@ impl Queue { self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; + if let Some(ticket) = front.ticket { + crate::g17::recovery::note_retired(); + crate::g17::trace::complete( + crate::g17::trace::COMPUTE, + ticket.qid, + ticket.ordinal, + &front.packet, + [timestamps[0], timestamps[1]], + ); + } self.retire_pending = true; defer(Deferred::Retired( front.packet.completion.clone(), @@ -906,6 +1076,9 @@ impl Queue { } } Ok(None) | Err(EAGAIN) => { + if restart.is_some() { + self.active.iter_mut().next().ok_or(EIO)?.observed_at = restart; + } self.retire_pending = self.front_finished(); break; } @@ -956,7 +1129,7 @@ impl Queue { if !retirement::quarantined_retired( &newest, self.graph.cursors()?, - self.graph.stamp()?, + self.witness_stamp()?, self.active.len as u32, self.submitted, )? { @@ -1000,6 +1173,7 @@ impl Queue { drop(self.active.pop()); } self.awaiting_witness = false; + self.fault_stamp = None; self.retirement_proved = false; self.replay_retirement = false; self.spared_deferred = false; @@ -1102,7 +1276,9 @@ impl Queue { /// Validate the complete unpublished item budget before changing descriptors, /// work counts or queue producers. Leave one item empty across ring wrap. fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { - if cursors.count != ITEM_RING_ENTRIES { return Err(EIO); } + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { return Err(ERANGE); } @@ -1111,14 +1287,18 @@ fn item_capacity(cursors: Cursors, entry_signal: bool) -> Result { let _ = crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeItems); return Err(EAGAIN); } - Ok(()) + Ok(()) } /// Retained commands are capacity, not a failed VM. Nothing has been written /// when this check runs, so the outer publisher can wait for retirement under /// its original admission deadline, just as it does for a full firmware ring. fn retained_capacity( - quarantined: bool, released: bool, active: usize, replays: usize, replay_front: bool, + quarantined: bool, + released: bool, + active: usize, + replays: usize, + replay_front: bool, ) -> Result { if quarantined || released { return Err(EBUSY); @@ -1139,17 +1319,68 @@ fn release_pending(released: bool, quarantined: bool, active: usize, retiring: b mod admission_tests { use super::*; + #[test] + fn complete_native_wait_list_fits_every_cached_slot() { + let waits = crate::g17::kick::COMPUTE_CHANNELS; + for slot in 0..RECORD_SLOTS { + for index in 0..waits { + let start = record_start( + slot, + NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, + CACHED_RECORD_STRIDE, + ) + .unwrap(); + assert!(start >= slot * CACHED_RECORD_STRIDE); + assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); + } + } + } + + #[test] + fn record_window_cannot_overwrite_the_next_slot() { + // The nineteenth wait used to cross the old slot boundary silently. + let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; + assert_eq!( + record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), + Err(EINVAL) + ); + assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); + assert_eq!( + record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), + Err(EINVAL) + ); + assert_eq!( + record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), + Ok(CACHED_RECORD_STRIDE - 1) + ); + } + #[test] fn retained_ring_pressure_waits_without_discarding_work() { let mut active = Fifo::new(); for sequence in 0..IN_FLIGHT { active.push(sequence).unwrap(); } - assert_eq!(retained_capacity(false, false, active.len, 0, false), Err(EAGAIN)); + assert_eq!( + retained_capacity(false, false, active.len, 0, false), + Err(EAGAIN) + ); assert_eq!(active.front(), Some(&0)); assert_eq!(active.len, IN_FLIGHT); assert_eq!(active.pop(), Some(0)); - assert_eq!(retained_capacity(false, false, active.len, 0, false), Ok(())); + assert_eq!( + retained_capacity(false, false, active.len, 0, false), + Ok(()) + ); assert_eq!(active.front(), Some(&1)); } @@ -1157,28 +1388,114 @@ mod admission_tests { fn replay_front_and_terminal_ownership_remain_distinct() { assert_eq!(retained_capacity(false, false, 0, 1, false), Err(EAGAIN)); assert_eq!(retained_capacity(false, false, 0, 1, true), Ok(())); - assert_eq!(retained_capacity(false, false, IN_FLIGHT, 1, true), Err(EAGAIN)); + assert_eq!( + retained_capacity(false, false, IN_FLIGHT, 1, true), + Err(EAGAIN) + ); for (quarantined, released) in [(true, false), (false, true), (true, true)] { - assert_eq!(retained_capacity(quarantined, released, 0, 1, true), Err(EBUSY)); + assert_eq!( + retained_capacity(quarantined, released, 0, 1, true), + Err(EBUSY) + ); } } #[test] fn item_refusal_is_retryable_and_preserves_the_empty_slot() { let n = ITEM_RING_ENTRIES; - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, false), Ok(())); - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 4, count: n }, true), Err(EAGAIN)); - assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, false), Ok(())); - assert_eq!(item_capacity(Cursors { producer: n - 2, consumer: 2, count: n }, true), Err(EAGAIN)); - assert_eq!(item_capacity(Cursors { producer: 10, consumer: 10, count: n }, true), Ok(())); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 4, + count: n + }, + false + ), + Ok(()) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 4, + count: n + }, + true + ), + Err(EAGAIN) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n - 2, + consumer: 2, + count: n + }, + false + ), + Ok(()) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n - 2, + consumer: 2, + count: n + }, + true + ), + Err(EAGAIN) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 10, + consumer: 10, + count: n + }, + true + ), + Ok(()) + ); } #[test] fn bad_item_cursors_are_not_backpressure() { let n = ITEM_RING_ENTRIES; - assert_eq!(item_capacity(Cursors { producer: 0, consumer: 0, count: 0 }, false), Err(EIO)); - assert_eq!(item_capacity(Cursors { producer: n, consumer: 0, count: n }, false), Err(ERANGE)); - assert_eq!(item_capacity(Cursors { producer: 0, consumer: n, count: n }, true), Err(ERANGE)); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: 0, + count: 0 + }, + false + ), + Err(EIO) + ); + assert_eq!( + item_capacity( + Cursors { + producer: n, + consumer: 0, + count: n + }, + false + ), + Err(ERANGE) + ); + assert_eq!( + item_capacity( + Cursors { + producer: 0, + consumer: n, + count: n + }, + true + ), + Err(ERANGE) + ); } #[test] @@ -1214,7 +1531,9 @@ impl Queue { Ok(true) } pub(crate) fn reusable(&self) -> bool { - if self.growing { return false; } + if self.growing { + return false; + } self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) @@ -1230,11 +1549,15 @@ impl Queue { && self.active.len == 0 && self.replays.len == 0 && self.previous.is_none() + // Borrow acquisition is serialized by the device mutex. Avoid + // touching the separate loan counter for an otherwise busy queue. + && !self.source_hint.has_readers(&self.source_readers) } pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { if !self.reusable() { return Err(EBUSY); } + self.source_hint.reset_for_owner(&self.source_readers)?; if !context.is_current() { return Err(EFAULT); } @@ -1274,6 +1597,13 @@ impl Queue { self.retained_owner = None; } context.mark_published(); + context.note_compute_binding(true); + crate::g17::trace::bind( + crate::g17::trace::COMPUTE, + self.qid(), + context.id(), + if same_vm { 2 } else { 1 }, + ); self.owner = Some(owner); self.unpublished_binding = true; self.released = false; @@ -1284,6 +1614,9 @@ impl Queue { /// True allows the registry to cancel the never-published QID and drop this graph. An /// installed graph remains retained even after the logical owner exits without work. pub(crate) fn release_owner(&mut self) -> Result { + if let Some(binding) = &self.binding { + binding.context.note_compute_binding(false); + } if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { return Ok(true); } @@ -1338,7 +1671,10 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() + self.pool_phase == PoolPhase::Populated + && self.pool.populated() + && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } /// A recycled killed queue still holding the backing its killed work used. pub(crate) fn recycled_pending(&self) -> bool { @@ -1396,6 +1732,7 @@ impl Queue { } fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; + self.config_cache = None; self.exit = None; self.quarantined = false; self.quarantine_error = None; @@ -1405,6 +1742,7 @@ impl Queue { self.spared_deferred = false; self.retire_pending = false; self.awaiting_witness = false; + self.fault_stamp = None; self.retirement_proved = false; self.replay_retirement = false; self.replay_error = None; @@ -1430,9 +1768,8 @@ impl Queue { if !self.vacate_candidate() { return Err(EBUSY); } - let record = ControlRecord::FreeListRelease( - self.pool.release_record(self.free_list_generation), - ); + let record = + ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); self.release_sent_ns = Some(crate::g17::now_ns()); @@ -1447,7 +1784,9 @@ impl Queue { consumer: u32, producer: u32, ) -> Result> { - if self.growing { return Ok(None); } + if self.growing { + return Ok(None); + } let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { return Ok(None); }; @@ -1456,11 +1795,18 @@ impl Queue { } // The retained owner binding's client alias pins the backing: detach it // first (the owner has exited; `reusable()` proved no previous binding). - let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); + let alias = self + .binding + .as_mut() + .and_then(|binding| binding.pool_alias.take()); let pages = self.pool.take_pages(); - let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); + let descriptors = self + .binding + .as_mut() + .and_then(|binding| binding.descriptors.take()); let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; + self.pool_phase = PoolPhase::Vacant; + self.release_sent_ns = None; Ok(Some(Vacated { _alias: alias, _pages: pages, @@ -1475,7 +1821,9 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if !self.can_rearm() { return Err(EAGAIN); } + if !self.can_rearm() { + return Err(EAGAIN); + } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; @@ -1489,27 +1837,50 @@ impl Queue { /// Only a capacity increase waits for this queue's previously published work. /// Other queues and ordinary submissions keep their existing concurrency. - pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { - if self.growing { return Err(EAGAIN); } - if self.quarantined || self.released || self.exit.is_some() - || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() + pub(crate) fn begin_growth( + &mut self, + init: &crate::g17::initdata::InitData, + blocks: usize, + ) -> Result { + if self.growing { + return Err(EAGAIN); + } + if self.quarantined + || self.released + || self.exit.is_some() + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() || (self.installed && !self.retirement_ready) - || self.pool_phase != PoolPhase::Populated { return Err(EAGAIN); } + || self.pool_phase != PoolPhase::Populated + { + return Err(EAGAIN); + } if self.installed { - let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let record = + ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); let cursor = Rings::publish_control(init, &record)?; self.pool_phase = PoolPhase::ReleaseSent(cursor); - self.release_sent_ns = Some(crate::g17::now_ns()); - } else { self.pool_phase = PoolPhase::Vacant; self.release_sent_ns = None; } + self.release_sent_ns = Some(crate::g17::now_ns()); + } else { + self.pool_phase = PoolPhase::Vacant; + self.release_sent_ns = None; + } self.growing = true; self.growth_blocks = blocks; Ok(self.installed) } - pub(crate) fn finish_growth(&mut self, init: &crate::g17::initdata::InitData, + pub(crate) fn finish_growth( + &mut self, + init: &crate::g17::initdata::InitData, pages: &mut Option, - alias: &mut Option, blocks: usize) -> Result { - if !self.growing || blocks < self.growth_blocks { return Err(EAGAIN); } + alias: &mut Option, + blocks: usize, + ) -> Result { + if !self.growing || blocks < self.growth_blocks { + return Err(EAGAIN); + } if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { let consumer = init.control_consumer()?.load(Ordering::Acquire); let producer = init.control_producer()?.load(Ordering::Acquire); @@ -1517,7 +1888,9 @@ impl Queue { return Err(EAGAIN); } } - if alias.is_none() { return Err(EINVAL); } + if alias.is_none() { + return Err(EINVAL); + } let fresh = pages.as_ref().ok_or(EINVAL)?; let binding = self.binding.as_mut().ok_or(EFAULT)?; // No ownership moves before every fallible descriptor/header operation. @@ -1529,11 +1902,19 @@ impl Queue { self.release_sent_ns = None; self.growing = false; self.growth_blocks = 0; - Ok(Vacated { _alias: old_alias, _pages: pages.take(), _descriptors: None, _ring: None }) + Ok(Vacated { + _alias: old_alias, + _pages: pages.take(), + _descriptors: None, + _ring: None, + }) } pub(crate) fn growth_token(&self) -> Option { - self.growing.then_some(Growth { qid: self.qid(), phase: self.pool_phase }) + self.growing.then_some(Growth { + qid: self.qid(), + phase: self.pool_phase, + }) } /// Transfer an abandoned builder's release back to the event worker. Do not @@ -1634,11 +2015,11 @@ impl Queue { let producer = self .graph .graph - .word(POINTERS + self.graph.pointers.producer)? + .word(CURSORS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + target)? + .word(CURSORS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1675,9 +2056,7 @@ impl Queue { let ticket = active.ticket.ok_or(EIO)?; let state = crate::g17::job::fence_status(&active.packet.completion.fence()); if state < 0 { - return Err(crate::g17::dependency::inherited_error(Error::from_errno( - state, - ))); + return Err(crate::g17::dependency::PRODUCER_FAILED); } if state > 0 { return Ok(None); @@ -1690,6 +2069,17 @@ impl Queue { } } +/// Validate the individual slot before deriving an offset into its allocation. +/// Allocation-wide checks cannot detect a write into the following live slot. +fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Result { + if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { + return Err(EINVAL); + } + slot.checked_mul(stride) + .and_then(|start| start.checked_add(offset)) + .ok_or(EINVAL) +} + impl Graph { fn record( &mut self, @@ -1699,12 +2089,15 @@ impl Graph { value: T, ) -> Result { let (object, start) = if let Some(slot) = slot { - if slot >= RECORD_SLOTS { - return Err(EINVAL); - } - (&mut self.records, slot * RECORD_STRIDE + offset) + let (object, stride): (&mut KernelObject, usize) = if offset == ANNOUNCE { + (&mut self.records, RECORD_STRIDE) + } else { + (&mut self.cached_records, CACHED_RECORD_STRIDE) + }; + (object, record_start(slot, offset, window, stride)?) } else { - (&mut self.graph, 0x6800 + offset) + let start = record_start(0, offset, window, GRAPH_SIZE - 0x6800)?; + (&mut *self.graph, 0x6800 + start) }; if core::mem::size_of::() > window { return Err(EINVAL); @@ -1730,7 +2123,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + self.pointers.producer)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -1756,8 +2149,7 @@ pub(crate) trait Host { ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) - -> Result; + fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; } @@ -1775,6 +2167,17 @@ impl Queue { dependencies: &[crate::g17::fw::kick::KickDependency], host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.publish_with_native_waits(packet, dependencies, &[], host, defer) + } + + pub(in crate::g17) fn publish_with_native_waits( + &mut self, + packet: Arc, + dependencies: &[crate::g17::fw::kick::KickDependency], + native: &[crate::g17::sync_input::NativeWait], + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { use crate::g17::fw::{ channels::WorkSlot, @@ -1809,11 +2212,26 @@ impl Queue { let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; + // The first prototype only waits on an installed consumer channel. + // A retry without a published source falls back to completion on CPU. + if first && !native.is_empty() { + return Err(EAGAIN); + } + if native.len() > crate::g17::kick::COMPUTE_CHANNELS { + return Err(EINVAL); + } let entry_signal = first || dependencies .iter() .any(|dependency| dependency.qid() != qid); item_capacity(self.graph.cursors()?, entry_signal)?; + if !native.is_empty() { + let cursors = self.graph.cursors()?; + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < (if entry_signal { 4 } else { 3 }) + native.len() as u32 { + return Err(EAGAIN); + } + } let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -1858,6 +2276,10 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; + let cached = self + .config_cache + .and_then(|(config, at)| (at == epoch).then_some(config)); + let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -1879,6 +2301,7 @@ impl Queue { priority: policy.priority(), }; let published = host.next_compute_publication()?; + self.last_published = published; self.active .push(Active { packet: packet.clone(), @@ -1904,7 +2327,7 @@ impl Queue { .object .initialize::( kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write(&args), + |descriptor| descriptor.write_zeroed(&args), )?; if first { self.graph.context_high.write( @@ -1921,8 +2344,26 @@ impl Queue { let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } - let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; + let config_va = if send_config { + Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) + } else { + None + }; let cursors = self.graph.cursors()?; + let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; + for (index, wait) in native.iter().enumerate() { + let record = wait.bytes( + crate::g17::fw::stamp_value(kick_timestamp.get() as u32), + packet.order.sequence, + ); + let address = self.graph.record( + record_slot, + NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, + record, + )?; + wait_vas.push(address, GFP_KERNEL)?; + } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; if first { @@ -1948,7 +2389,7 @@ impl Queue { })?; producer = self .graph - .append(producer, config_va, || {}) + .append(producer, config_va.ok_or(EINVAL)?, || {}) .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; @@ -1957,15 +2398,25 @@ impl Queue { })?; } else { let target = self.pool.reserve_submission()?; - producer = self - .graph - .append(cursors.producer, descriptor_high, || { - packet.mark_published() + producer = (|| -> Result { + let mut before_descriptor = cursors.producer; + for &wait in &wait_vas { + before_descriptor = self.graph.append(before_descriptor, wait, || { + packet.mark_published(); + visible = true; + })?; + } + self.graph.append(before_descriptor, descriptor_high, || { + packet.mark_published(); + visible = true; }) - .or_else(|error| { + })() + .or_else(|error| { + if !visible { self.pool.cancel_unpublished(target)?; - Err(error) - })?; + } + Err(error) + })?; visible = true; self.unpublished_binding = false; let qos = host.qos_publish( @@ -1978,14 +2429,16 @@ impl Queue { policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; - producer = self - .graph - .append(producer, config_va, || {}) - .or_else(|error| { - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; + if let Some(config_va) = config_va { + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; + } } if first { prepared_kick.prepare_compute_install()?; @@ -2015,10 +2468,12 @@ impl Queue { self.outer_started & ring == 0, )?, )?; - let notify_activation = self.outer_started & ring == 0; self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); + if send_config { + self.config_cache = Some((config, epoch)); + } self.ordinal = ordinal; let (accepted, state) = host.epoch()?; if state != 0 || accepted != epoch { @@ -2043,13 +2498,13 @@ impl Queue { previous.publication = Some((packet.order.sequence, epoch)); } } - if notify_activation { - fence(Ordering::SeqCst); - // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; - } + // Every publication rings its own pipe and priority class, as render + // publication does: pipe 2 (compute) in bits 1:0, priority class in + // bits 3:2. The generic work doorbell left the firmware to find the + // ring itself and delayed dispatch. fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x10)?; + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); if let Err(error) = result { @@ -2087,8 +2542,13 @@ impl Queue { .replays .front() .is_some_and(|front| Arc::ptr_eq(front, packet)); - retained_capacity(self.quarantined, self.released, self.active.len, - self.replays.len, replay_front)?; + retained_capacity( + self.quarantined, + self.released, + self.active.len, + self.replays.len, + replay_front, + )?; if !packet.context.is_current() || !self .binding @@ -2155,7 +2615,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + self.pointers.producer)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); @@ -2176,17 +2636,30 @@ impl Queue { u8::from(self.quarantined), u8::from(self.awaiting_witness), u8::from(self.previous.is_some()), u8::from(self.exit.is_some()), u8::from(self.growing), self.pool_phase, self.active.len, IN_FLIGHT)?; match self.graph.cursors() { - Ok(cursors) if cursors.count != 0 => writeln!(f, "items_used={} items_total={}", - cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, cursors.count - 1), + Ok(cursors) if cursors.count != 0 => writeln!( + f, + "items_used={} items_total={}", + cursors.producer.wrapping_sub(cursors.consumer) % cursors.count, + cursors.count - 1 + ), _ => writeln!(f, "items_unavailable=1"), }?; - writeln!(f, "compute_settling qid={} release={} pool_release={}", self.qid(), - u8::from(self.release_settling()), u8::from(self.pool_release_pending()))?; + writeln!( + f, + "compute_settling qid={} release={} pool_release={}", + self.qid(), + u8::from(self.release_settling()), + u8::from(self.pool_release_pending()) + )?; if let Some(binding) = self.binding.as_ref() { - binding.context.write_capacity(f, "compute", usize::from(self.qid()))?; + binding + .context + .write_capacity(f, "compute", usize::from(self.qid()))?; } if let Some(previous) = self.previous.as_ref() { - previous.context.write_capacity(f, "compute_previous", usize::from(self.qid()))?; + previous + .context + .write_capacity(f, "compute_previous", usize::from(self.qid()))?; } Ok(()) } @@ -2262,21 +2735,66 @@ impl Queue { /// Logical queue/context ownership is unchanged. The existing binding is /// retained through the successor's exact replacement witness. pub(crate) fn share_idle(&mut self) -> Result { - if !self.installed || self.owner.is_none() || !self.retirement_ready - || self.growing || self.quarantined || self.awaiting_witness - || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 - || self.previous.is_some() || self.pool_phase != PoolPhase::Populated - { + if !self.shareable() { return Ok(false); } self.release_owner()?; Ok(self.reusable()) } + + /// Another owner could take this channel now: idle and released, shareable, or vacant. + pub(crate) fn channel_free(&self) -> bool { + self.reusable() || self.shareable() || self.can_rearm() + } + + /// A pool replacement is in flight for this queue. + pub(crate) fn growing(&self) -> bool { + self.growing + } + + /// The queue's first command has been published: its identity and configuration exist. + pub(crate) fn installed(&self) -> bool { + self.installed + } + + /// A packet of the bound owner passed prepare_job and will publish here. + pub(crate) fn claim(&mut self) { + self.claims = self.claims.saturating_add(1); + } + + pub(crate) fn release_claim(&mut self) { + self.claims = self.claims.saturating_sub(1); + } + + /// An open client's idle channel that [`Self::share_idle`] can hand over. + pub(crate) fn shareable(&self) -> bool { + self.claims == 0 + && self.installed + && self.owner.is_some() + && self.retirement_ready + && !self.growing + && !self.quarantined + && !self.awaiting_witness + && self.exit.is_none() + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + && self.pool_phase == PoolPhase::Populated + && !self.source_hint.has_readers(&self.source_readers) + } + + /// Device publication order of this channel's newest command: lower is + /// idle longer. + pub(crate) fn last_published(&self) -> u64 { + self.last_published + } } impl Queue { pub(crate) fn can_rearm(&self) -> bool { - self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + self.pool_phase == PoolPhase::Vacant + && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } } @@ -2286,19 +2804,35 @@ impl Queue { /// caller validates the current preparation epoch and cancels only a true /// result's never-published QID; installed rings and identities stay intact. pub(crate) fn rollback_unpublished( - &mut self, owner: u64, context: &Arc, + &mut self, + owner: u64, + context: &Arc, ) -> Result<(bool, Option)> { - if self.owner != Some(owner) || !self.unpublished_binding - || self.binding.as_ref().is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) - || self.growing || self.quarantined || self.awaiting_witness - || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 - || self.retire_pending || self.failure_status_pending + if self.owner != Some(owner) + || !self.unpublished_binding + || self + .binding + .as_ref() + .is_none_or(|binding| !Arc::ptr_eq(&binding.context, context)) + || self.growing + || self.quarantined + || self.awaiting_witness + || self.exit.is_some() + || self.active.len != 0 + || self.replays.len != 0 + || self.retire_pending + || self.failure_status_pending || self.pool_phase != PoolPhase::Populated - || self.previous.as_ref().is_some_and(|previous| previous.publication.is_some()) + || self + .previous + .as_ref() + .is_some_and(|previous| previous.publication.is_some()) { return Ok((false, None)); } - if !self.installed { return Ok((true, None)); } + if !self.installed { + return Ok((true, None)); + } let mut retired = None; if let Some(previous) = self.previous.as_ref() { self.graph.set_owner(&previous.context)?; @@ -2318,3 +2852,83 @@ impl Queue { Ok((false, retired)) } } + +impl Queue { + /// Same immutable singleton identity and current physical epoch. + pub(crate) fn has_input_source( + &self, + source: &Arc, + status: &Arc, + epoch: u64, + ) -> bool { + !self.quarantined + && self.replays.len == 0 + && self.active.iter().any(|active| { + active.ticket.is_some_and(|ticket| ticket.epoch == epoch) + && Arc::ptr_eq(active.packet.completion.status(), status) + && active.packet.completion.is_input_source(source) + }) + } + + pub(crate) fn source_dependency( + &mut self, + source: &crate::g17::sync_input::Source, + status: &Arc, + epoch: u64, + ) -> Result> { + for active in self.active.iter() { + if !active.packet.completion.is_input_source(&source.submission) { + continue; + } + if !Arc::ptr_eq(active.packet.completion.status(), status) { + return Err(EINVAL); + } + if self.quarantined || self.replays.len != 0 { + return Err(EAGAIN); + } + let ticket = active.ticket.ok_or(EAGAIN)?; + if ticket.epoch != epoch { + return Err(EAGAIN); + } + self.valid_ticket(active, &ticket)?; + if active.packet.cancel_requested() { + return Err(ECANCELED); + } + source.submission.source_status()?; + let dependency = crate::g17::fw::kick::KickDependency::new( + self.qid(), + crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?, + ) + .ok_or(EIO)?; + let hold = self.source_hint.hold(&self.source_readers)?; + return Ok(Some(( + active.published, + dependency, + self.graph.support.gpu_va() + STAMP as u64, + u32::from(self.qid()), + hold, + ))); + } + Ok(None) + } + + /// Preparing a possible wait does not create a firmware consumer. Mark it + /// only once the consumer's first item is visible, under the device mutex. + pub(crate) fn note_input_consumer(&self, source: &crate::g17::sync_input::Source) { + for active in self.active.iter() { + if active.packet.completion.is_input_source(&source.submission) { + active.packet.completion.note_firmware_consumer(); + break; + } + } + } +} + +/// Publication, dependency, stamp address, queue and the retained source hold. +pub(crate) type SourceDependency = ( + u64, + crate::g17::fw::kick::KickDependency, + u64, + u32, + crate::g17::sync_input::Hold, +); diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index d7386e335e167..719a74787ca70 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -83,11 +83,12 @@ impl Transaction { } impl Queue { - /// A command with ray traversal scratch is never killed. On Neo, killing a + /// A running command with ray traversal scratch is never killed. On Neo, killing a /// traversal that stopped making progress either crashed the GPU firmware /// when the queue's pool was then released, or stalled every later render /// when it was retained; left alone, the rest of the GPU kept working. Such - /// a queue stays quarantined and retained until its commands retire. + /// a queue stays quarantined and retained until its commands retire. A + /// traversal the firmware reported failed no longer runs (see `fault_pending`). fn active_traversal(&self) -> bool { self.active.iter().any(|active| { matches!( @@ -101,7 +102,7 @@ impl Queue { && self.released && self.active.len != 0 && self.exit.is_none() - && !self.active_traversal() + && (!self.active_traversal() || self.fault_pending()) && (self.quarantined || self.retired_by_teardown) && age >= if self.quarantined { @@ -110,6 +111,10 @@ impl Queue { CANCELLED_DELAY_NS } } + /// A kill of `context` was published through this queue. + pub(crate) fn exit_published(&self, context: &Arc) -> bool { + self.exit.is_some() && self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + } pub(crate) fn exit_owner(&self, context: &Arc) -> Option { if !self .active @@ -124,6 +129,23 @@ impl Queue { .and_then(|active| active.ticket.map(|ticket| ticket.owner)) }) } + /// The kill of this queue's failed context waits; returns how long it has waited, and + /// whether this is the first time it crossed `slow`. + pub(crate) fn note_kill_wait(&mut self, now: u64, slow: u64) -> (u64, bool) { + let (since, warned) = self.kill_wait.get_or_insert((now, false)); + let waited = now.saturating_sub(*since); + let crossed = waited >= slow && !*warned; + *warned |= crossed; + (waited, crossed) + } + /// How long the kill of this queue's failed context has been waiting, if it is. + pub(crate) fn kill_waiting(&self, now: u64) -> Option { + self.kill_wait.map(|(since, _)| now.saturating_sub(since)) + } + /// The kill was published: how long it waited, if it did. + pub(crate) fn take_kill_wait(&mut self, now: u64) -> u64 { + self.kill_wait.take().map_or(0, |(since, _)| now.saturating_sub(since)) + } pub(crate) fn record_exit(&mut self, cookie: u64, now: u64) { self.exit = Some(Transaction { cookie, @@ -162,6 +184,7 @@ impl Queue { return false; } exit.phase = Phase::Acknowledged; + crate::g17::trace::teardown(4, cookie, u64::from(self.qid()), 0); true } pub(crate) fn settle_exit( diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index e938ec566dbac..d7f821bbce68d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,7 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. -use crate::g17::initdata::ComputeCompletion; +use crate::g17::initdata::{CompletionFold, ComputeCompletion}; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -52,6 +52,12 @@ impl Cursors { } } +/// Whether stamp `a` was stored after stamp `b` (both kick stamps, modulo wrap). +pub(crate) const fn stamp_newer(a: u32, b: u32) -> bool { + let delta = a.wrapping_sub(b); + delta != 0 && delta < 0x8000_0000 +} + /// Outstanding kicks span fewer than half the 24-bit stamp sequence. Zero is a valid wrap. pub(crate) const fn stamp_covers(word: u32, kick: u64) -> bool { let delta = word.wrapping_sub((kick as u32).wrapping_shl(8)); @@ -66,15 +72,16 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone. +/// before it retires on the record alone, counted from when its ioctl's work-state node drained. /// /// The firmware stores the stamp after the cache maintenance that follows the record, so the /// stamp, not the record, proves that the command's writes are complete and that its mappings -/// may be released. While other commands of the same submit ioctl are still running, the stamp -/// routinely lands several milliseconds after the record (on J700 under games, up to about 3 ms -/// after the 1 ms this used to wait). Only a stamp still missing well past that is treated as -/// never stored. The bound stays far below the scheduler timeout, so such a command cannot fail -/// its VM. +/// may be released. It stores the stamps of an ioctl's commands only once every kick counted on +/// the ioctl's work-state node has completed: while other commands of the same submit ioctl +/// still run, the stamp is not late, however long they run (a command buffer of a dozen render +/// passes, or a render waiting on another queue). [`stamp_wait`] restarts the bound until the +/// node drains. Only a stamp still missing well past that is treated as never stored. The bound +/// stays far below the scheduler timeout, so such a command cannot fail its VM. pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; // A command that waits out the fallback must still retire well inside the scheduler timeout. @@ -86,15 +93,46 @@ pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) } +/// Where a command's wait for its completion stamp stands; see [`stamp_wait`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum StampWait { + /// Within [`STAMP_FALLBACK_NS`] of the record, or no record yet. + Waiting, + /// The bound passed while the ioctl's work-state node still runs kicks: the stamp is not + /// late, and the wait counts again from the given host time. + Restart(u64), + /// The node drained and the stamp is still missing after the bound: retire on the record. + Aged, +} + +/// The stamp wait of a command whose exact record was latched at `observed_at` (host ns) and +/// whose stamp is still missing, at `now`. `drained` reports whether every kick counted on the +/// command's work-state node completed; it is asked only once the bound has passed. +pub(crate) fn stamp_wait(observed_at: Option, now: u64, drained: impl FnOnce() -> bool) + -> StampWait +{ + if !record_aged(observed_at, now) { + StampWait::Waiting + } else if drained() { + StampWait::Aged + } else { + StampWait::Restart(now) + } +} + static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(crate) fn stampless_total() -> u64 { + STAMPLESS.load(core::sync::atomic::Ordering::Relaxed) +} + /// Counts a command that retired on its record because its completion stamp never arrived; /// logs the first and every 64th. pub(crate) fn note_stampless(qid: u8, kick: u64) { let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; if count == 1 || count % 64 == 0 { pr_info!( - "G17: {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", + "G17: lifetime total {} compute commands retired on their completion record after {} ms without a completion stamp (latest: queue {} kick {:#x})\n", count, STAMP_FALLBACK_NS / 1_000_000, qid, @@ -106,7 +144,9 @@ pub(crate) fn note_stampless(qid: u8, kick: u64) { /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. /// The command is complete when its stamp covers it, or when its exact completion record has -/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`); in both cases its items +/// been present for [`STAMP_FALLBACK_NS`] without one (`record_aged`) in the same recovery +/// generation. Recovery may consume items without completing their writes, so an old record +/// and consumed items cannot replace the stamp across a restart. In both cases its items /// must be consumed. #[expect( clippy::too_many_arguments, @@ -125,7 +165,9 @@ pub(crate) fn poll( if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { return Err(ENODATA); } - let completed = stamp_covers(stamp, ticket.kick) || (observed.is_some() && record_aged); + let same_generation = !matches!(recordless, Recordless::CrossedRecovery); + let completed = stamp_covers(stamp, ticket.kick) + || (same_generation && observed.is_some() && record_aged); if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { return Ok(None); } @@ -161,35 +203,122 @@ pub(crate) fn quarantined_retired( && submitted == pending) } +/// Walks a completion ring of `seen.len()` slots from `start` over the records written since +/// the previous walk: `seen` holds each slot's control word as of the last walk that passed it, +/// so the first slot still holding it is where the firmware writes next. `record` folds one +/// slot; the walk also ends once it reports every request found, or at a requested record +/// still being written, which the next walk reads again. Returns where the next walk starts. +pub(crate) fn walk_completions( + start: usize, + seen: &mut [u64], + mut control: impl FnMut(usize) -> Result, + mut record: impl FnMut(usize) -> Result, +) -> Result { + let slots = seen.len(); + if slots == 0 { + return Err(EINVAL); + } + let start = start % slots; + for step in 0..slots { + let slot = (start + step) % slots; + let word = control(slot)?; + if word == seen[slot] { + return Ok(slot); + } + match record(slot)? { + CompletionFold::Incomplete => return Ok(slot), + CompletionFold::Passed => seen[slot] = word, + CompletionFold::Done => { + seen[slot] = word; + return Ok((slot + 1) % slots); + } + } + } + Ok(start) +} + #[derive(Clone, Copy)] struct Request { descriptor: u64, kick: u64, observed: Option, + covered: bool, + next: u16, +} + +const NO_REQUEST: u16 = u16::MAX; +const _: () = assert!(QIDS * IN_FLIGHT < NO_REQUEST as usize); + +fn request_bucket(qid: usize, kick: u64) -> usize { + qid * IN_FLIGHT + kick as usize % IN_FLIGHT } /// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; /// staging and folding never allocate under the device mutex. pub(crate) struct Batch { requests: KVVec, + /// Per-QID kick buckets; collisions retain the complete identity check. + heads: KVVec, slots: [(usize, usize); QIDS], open: Option, + /// Requests with no observation yet; the scan stops when this reaches zero. + unobserved: usize, + /// Unobserved requests whose queue's completion stamp already covered them when staged. + /// The firmware stores the stamp after the record, so each has a record in the ring. + covered: usize, + /// Per QID, the next kick its queue had not published when the batch was staged + /// ([`NO_FRONTIER`] for a QID without a staged queue). A record at or past it belongs + /// to a command published during the scan, which a later batch stages. + frontiers: [u64; QIDS], } +const NO_FRONTIER: u64 = u64::MAX; +const KICK_MASK: u64 = (1 << 40) - 1; + impl Batch { pub(crate) fn new() -> Result { + let mut heads = KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?; + for _ in 0..QIDS * IN_FLIGHT { + heads.push_within_capacity(NO_REQUEST).map_err(|_| ENOSPC)?; + } Ok(Self { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, + heads, slots: [(0, 0); QIDS], open: None, + unobserved: 0, + covered: 0, + frontiers: [NO_FRONTIER; QIDS], }) } pub(crate) fn clear(&mut self) { self.requests.clear(); self.slots.fill((0, 0)); self.open = None; + self.unobserved = 0; + self.covered = 0; + self.frontiers.fill(NO_FRONTIER); + } + /// Records the first kick `qid`'s queue had not yet published when this batch was staged. + pub(crate) fn stage_frontier(&mut self, qid: u8, next: u64) -> Result { + *self.frontiers.get_mut(usize::from(qid)).ok_or(EINVAL)? = next & KICK_MASK; + Ok(()) + } + /// A record of `qid` with `kick` names a command published after this batch was staged. + fn published_after_staging(&self, qid: usize, kick: u64) -> bool { + let frontier = self.frontiers[qid]; + frontier != NO_FRONTIER && kick.wrapping_sub(frontier) & KICK_MASK < 1 << 39 + } + /// True once every requested identity has an observation. + pub(crate) fn complete(&self) -> bool { + self.unobserved == 0 + } + /// Some request whose record must already be in the ring is still unobserved. + pub(crate) fn covered_pending(&self) -> bool { + self.covered != 0 } - pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { + /// Stages `ticket`; `covered` when its queue's completion stamp already covers its kick. + pub(crate) fn request(&mut self, ticket: &Ticket, covered: bool) -> Result { if ticket.descriptor == 0 || ticket.kick == 0 { return Err(EINVAL); } @@ -201,65 +330,119 @@ impl Batch { return Err(ENOSPC); } let start = if slot.1 == 0 { + let begin = ticket.qid as usize * IN_FLIGHT; + self.heads.get_mut(begin..begin + IN_FLIGHT).ok_or(EIO)?.fill(NO_REQUEST); self.requests.len() } else { slot.0 }; + let bucket = request_bucket(ticket.qid as usize, ticket.kick); + let next = *self.heads.get(bucket).ok_or(EIO)?; + let index = u16::try_from(self.requests.len()).map_err(|_| ENOSPC)?; self.requests .push_within_capacity(Request { descriptor: ticket.descriptor, kick: ticket.kick, observed: None, + covered, + next, }) .map_err(|_| ENOSPC)?; + *self.heads.get_mut(bucket).ok_or(EIO)? = index; *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); + self.unobserved += 1; + self.covered += usize::from(covered); Ok(()) } - /// Capture the control word once, then read only a requested identity's payload. - /// Records can be overwritten during a scan: incomplete or older identities are skipped. - pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - let control = record.read(0)?; + /// Capture the control word once. Most slots belong to an older kick of a + /// selected QID; reject those before reading any additional shared words. + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + if !self.fold_reader(|word| record.read(word))? { + return Ok(CompletionFold::Incomplete); + } + Ok(self.folded()) + } + + /// Like [`Self::fold`], for a scan that reads every slot regardless of the outcome. + pub(crate) fn fold_any(&mut self, record: ComputeCompletion<'_>) -> Result { + self.fold_reader(|word| record.read(word)).map(|_| ()) + } + + fn folded(&self) -> CompletionFold { + if self.complete() { CompletionFold::Done } else { CompletionFold::Passed } + } + + /// Ok(false) when a pending request's record is still being written. + fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { + let control = read(0)?; let qid = ((control >> 9) & 0x7f) as usize; - let (start, count) = self.slots[qid]; + let (_, count) = self.slots[qid]; + let kick = + (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; if count == 0 { - return Ok(()); + return Ok(!self.published_after_staging(qid, kick)); } - let descriptor = record.read(6)? & !1; - if descriptor == 0 || record.read(7)? == 0 { - return Ok(()); + let mut index = *self.heads.get(request_bucket(qid, kick)).ok_or(EIO)?; + let mut descriptor = None; + let mut found = None; + for _ in 0..count { + if index == NO_REQUEST { + break; + } + let request = self.requests.get(usize::from(index)).ok_or(EIO)?; + if request.kick == kick { + let value = if let Some(value) = descriptor { + value + } else { + let value = read(6)? & !1; + if value == 0 || read(7)? == 0 { + return Ok(request.observed.is_some()); + } + descriptor = Some(value); + value + }; + if request.descriptor == value { + // Chains run newest first. Retain the original first-match + // rule even if a caller stages the same identity twice. + found = Some(usize::from(index)); + } + } + index = request.next; } - let kick = - (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; - let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; - let Some(request) = requests - .iter_mut() - .find(|r| r.descriptor == descriptor && r.kick == kick) - else { - return Ok(()); + let Some(index) = found else { + return Ok(!self.published_after_staging(qid, kick)); }; - let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; - if timestamps[0] == 0 - || timestamps[1] == 0 - || request - .observed - .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + let request = self.requests.get_mut(index).ok_or(EIO)?; + let timestamps = [read(2)? & TIME_MASK, read(5)? & TIME_MASK]; + if timestamps[0] == 0 || timestamps[1] == 0 { + return Ok(request.observed.is_some()); + } + if request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) { - return Ok(()); + return Ok(true); + } + if request.observed.is_none() { + self.unobserved -= 1; + self.covered -= usize::from(request.covered); } request.observed = Some(Observation { timestamps }); - Ok(()) + Ok(true) } + + /// A command published after this batch staged its queue's requests was not part of the + /// ring snapshot: it stays unobserved until a later batch. pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; - let request = self + Ok(self .requests .get_mut(start..start + count) .ok_or(EIO)? .iter_mut() .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) - .ok_or(EIO)?; - Ok(request.observed.take()) + .and_then(|request| request.observed.take())) } } @@ -306,6 +489,15 @@ mod tests { assert!(stamp_covers(0, 0xff_ffff)); } + #[test] + fn a_reported_fault_stamp_is_newer_only_until_stored() { + assert!(stamp_newer(own(KICK), own(KICK - 1))); + assert!(!stamp_newer(own(KICK), own(KICK))); + assert!(!stamp_newer(own(KICK - 1), own(KICK))); + // Across the wrap of the 24-bit kick in the stamp word. + assert!(stamp_newer(own(0), own(0xff_ffff))); + } + #[test] fn a_late_stamp_is_waited_for() { // Under games on J700 the stamp lands up to about 4 ms after the record while the @@ -374,6 +566,22 @@ mod tests { assert_eq!(result, Ok(Some([0x1111, 0x2222]))); } + #[test] + fn a_stamp_waits_for_the_rest_of_its_ioctl() { + let at = 1_000_000; + let late = at + STAMP_FALLBACK_NS; + // The node is asked only once the bound passed. + assert_eq!(stamp_wait(Some(at), late - 1, || panic!("node read early")), StampWait::Waiting); + assert_eq!(stamp_wait(None, u64::MAX, || panic!("node read without a record")), + StampWait::Waiting); + // Other kicks of the ioctl still run: the stamp is not late, the bound restarts. + assert_eq!(stamp_wait(Some(at), late, || false), StampWait::Restart(late)); + assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); + // The node drained and the stamp is still missing after the bound. + assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS, || true), StampWait::Aged); + assert_eq!(stamp_wait(Some(at), late, || true), StampWait::Aged); + } + #[test] fn an_aged_record_still_waits_for_consumed_items() { let record = Some(Observation { timestamps: [10, 20] }); @@ -386,6 +594,30 @@ mod tests { assert_eq!(result, Ok(None)); } + #[test] + fn recovery_cannot_turn_an_aged_record_into_a_release_witness() { + let record = Some(Observation { timestamps: [10, 20] }); + // Recovery can advance the consumer without completing cache maintenance. + // Even a record observed before the restart and aged past the fallback + // must retain its command's mappings until a real stamp covers it. + for stamp in [0, own(KICK - 1)] { + assert_eq!(poll(&ticket(), record, consumed(), stamp, + Recordless::CrossedRecovery, 0, None, true), Ok(None)); + } + // A real stamp restores the release witness; completion's interrupted + // classification independently prevents false successful fences. + for stamp in [own(KICK), own(KICK + 1)] { + assert_eq!(poll(&ticket(), record, consumed(), stamp, + Recordless::CrossedRecovery, 0, None, true), Ok(Some([10, 20]))); + } + let busy = Cursors { producer: 4, consumer: 2, count: 64 }; + assert_eq!(poll(&ticket(), record, busy, own(KICK), + Recordless::CrossedRecovery, 0, None, true), Ok(None)); + // The existing same-generation fallback remains available. + assert_eq!(poll(&ticket(), record, consumed(), own(KICK - 1), + Recordless::Allowed, 0, None, true), Ok(Some([10, 20]))); + } + #[test] fn a_zero_duration_record_fails() { let record = Some(Observation { timestamps: [5, 5] }); @@ -402,3 +634,156 @@ mod tests { assert_eq!(result, Err(ENODATA)); } } + + +#[kunit_tests(asahi_g17_completion_index)] +mod index_tests { + use super::*; + + fn ticket(qid: u8, kick: u64, descriptor: u64) -> Ticket { + Ticket { owner: 1, submission: 1, context_id: 1, qid, descriptor, + work_node: 0x2000, kick, ordinal: 1, item_producer: 4, item_count: 64, epoch: 1 } + } + fn record(ticket: &Ticket, timestamps: [u64; 2]) -> [u64; 8] { + let mut words = [0; 8]; + words[0] = ((ticket.kick & !0xff) << 8) | ((ticket.kick & 0xff) << 48) + | (u64::from(ticket.qid) << 9); + words[2] = timestamps[0]; words[5] = timestamps[1]; + words[6] = ticket.descriptor | 1; words[7] = 1; + words + } + #[test] + fn stale_kick_reads_only_the_control_even_on_a_bucket_collision() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); + let words = record(&ticket(7, 17, 0x1000), [10, 20]); + batch.fold_reader(|word| { + assert_eq!(word, 0); + Ok(words[word]) + }).unwrap(); + assert!(!batch.complete()); + } + #[test] + fn collisions_and_reused_descriptors_keep_full_kick_identity() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 49, 0x1000); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); + let words = record(&first, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(!batch.complete()); + let words = record(&second, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.take(&second).unwrap().unwrap().timestamps, [30, 40]); + } + #[test] + fn same_kick_still_checks_descriptor_and_zero_duration_is_retained() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 33, 0x2000); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); + let words = record(&first, [20, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [20, 20]); + assert!(batch.take(&second).unwrap().is_none()); + } + #[test] + fn duplicate_identity_keeps_the_first_staged_request() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t, false).unwrap(); batch.request(¤t, false).unwrap(); + let words = record(¤t, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(¤t).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.unobserved, 1); + } + #[test] + fn clear_restarts_buckets_and_later_publication_stays_unobserved() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); batch.clear(); + let current = ticket(7, 49, 0x1000); + batch.request(¤t, false).unwrap(); + let stale = record(&ticket(7, 33, 0x1000), [10, 20]); + batch.fold_reader(|word| Ok(stale[word])).unwrap(); + assert!(!batch.complete()); + assert!(batch.take(&ticket(7, 50, 0x2000)).unwrap().is_none()); + let words = record(¤t, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + } + #[test] + fn a_requested_record_still_being_written_is_read_again() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t, true).unwrap(); + let mut words = record(¤t, [10, 20]); + words[7] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + words[7] = 1; + words[5] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.covered_pending()); + let words = record(¤t, [10, 20]); + assert!(batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.complete() && !batch.covered_pending()); + } + #[test] + fn a_command_published_during_the_scan_is_left_for_the_next_batch() { + let mut batch = Batch::new().unwrap(); + batch.stage_frontier(7, 50).unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + batch.stage_frontier(9, 3).unwrap(); + let late = record(&ticket(7, 50, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(late[word])).unwrap()); + let idle = record(&ticket(9, 3, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(idle[word])).unwrap()); + let earlier = record(&ticket(9, 2, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(earlier[word])).unwrap()); + let wrapped = record(&ticket(7, 0xff_ffff_fff0, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(wrapped[word])).unwrap()); + } + fn walk(start: usize, seen: &mut [u64], ring: &[u64], outcome: impl Fn(usize) -> CompletionFold) + -> usize + { + walk_completions(start, seen, |slot| Ok(ring[slot]), |slot| Ok(outcome(slot))).unwrap() + } + #[test] + fn a_walk_reads_only_the_records_written_since_the_previous_one() { + let mut seen = [0u64; 8]; + let mut ring = [0u64; 8]; + assert_eq!(walk(0, &mut seen, &ring, |_| CompletionFold::Passed), 0); + ring[0] = 10; ring[1] = 11; ring[2] = 12; + let read = core::cell::Cell::new(0); + assert_eq!(walk(0, &mut seen, &ring, |_| { read.set(read.get() + 1); CompletionFold::Passed }), 3); + assert_eq!(read.get(), 3); + ring[3] = 13; + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + // A full lap later the firmware overwrote slots 4..8 and 0..2 with newer records. + for (slot, word) in [(4, 14), (5, 15), (6, 16), (7, 17), (0, 18), (1, 19)] { ring[slot] = word; } + assert_eq!(walk(4, &mut seen, &ring, |_| CompletionFold::Passed), 2); + } + #[test] + fn a_walk_stops_at_a_partial_record_and_after_the_last_request() { + let mut seen = [0u64; 8]; + let ring = [10, 11, 12, 13, 0, 0, 0, 0]; + assert_eq!(walk(0, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Incomplete } + else { CompletionFold::Passed }), 2); + assert_eq!(seen[..3], [10, 11, 0]); + assert_eq!(walk(2, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Done } + else { CompletionFold::Passed }), 3); + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + } + #[test] + fn an_unrequested_or_stale_record_is_passed() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + let mut stale = record(&ticket(7, 33, 0x1000), [10, 20]); + stale[7] = 0; + assert!(batch.fold_reader(|word| Ok(stale[word])).unwrap()); + let other = record(&ticket(9, 33, 0x1000), [10, 20]); + assert!(batch.fold_reader(|word| Ok(other[word])).unwrap()); + assert!(!batch.complete() && !batch.covered_pending()); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 6a9b15789339f..b7037a1c35f75 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -10,7 +10,8 @@ use super::{ command::{Parser, UscWindow, Validated}, completion::{Completion, Destinations, PreparedPublication}, context::Context, - fence::{Outputs, Submission}, + dependency::FailureRecord, + fence::Outputs, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, validation::Proof, }; @@ -27,7 +28,6 @@ use kernel::{ xarray, }; -const EARLY_DEPTH: usize = 2; static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); pub(crate) mod admission; @@ -70,7 +70,7 @@ impl Frontend { return Err(EINVAL); } let window = UscWindow::new(usc_base, range.clone())?; - let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + let fences = FenceContexts::new(1, c_str!("aurora_g17_queue"), FENCE_KEY)?; let allocation_failure = super::fence::allocation_failure(&fences)?; Ok(Self { fences, @@ -85,6 +85,7 @@ impl Frontend { pub(crate) struct Queue { dev: AsahiDevRef, context: Arc, + usage: Arc, backend: Arc, render: Option>, compute: Option>, @@ -96,13 +97,19 @@ pub(crate) struct Queue { proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], - failures: [Option<(u64, Fence)>; 2], + /// Finished frontier entries pruned during this ioctl. Their last references + /// can tear a whole completion down; that waits until the ioctl has published. + finished: KVec, + failures: [FailureRecord; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. _slot: admission::QueueSlot, } -type ValidatedBatch = (KVec<(Validated, Order, Destinations)>, KVec); +type ValidatedBatch = ( + KVec<(Validated, Order, Destinations)>, + KVec, +); impl Queue { pub(crate) fn new( @@ -111,6 +118,7 @@ impl Queue { backend: Arc, frontend: Frontend, slot: admission::QueueSlot, + usage: Arc, ) -> Result { let Frontend { fences, @@ -123,6 +131,7 @@ impl Queue { Ok(Self { dev: dev.into(), context, + usage, backend, render, compute: None, @@ -134,7 +143,8 @@ impl Queue { proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), - failures: [None, None], + finished: KVec::new(), + failures: [FailureRecord::new(), FailureRecord::new()], _slot: slot, }) } @@ -155,16 +165,25 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; + // Without room the entries are released here, as they always could be. + let mut finished = if self + .finished + .reserve(self.frontiers[index].len(), GFP_KERNEL) + .is_ok() + { + Some(&mut self.finished) + } else { + None + }; self.frontiers[index].retain(|(sequence, fences)| { - let status = job::fence_status(&fences.completed); + let status = fences.status(); if status == 0 { return true; } - // One earliest failed ordinal covers every later prefix without - // retaining its packet, mappings, or every failed fence. - if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { - *failure = Some((*sequence, fences.completed.clone())); + if let Some(finished) = finished.as_mut() { + let _ = finished.push_within_capacity(fences.clone()); } + failure.note(*sequence, status, fences); false }); } @@ -179,7 +198,7 @@ impl Queue { command: Validated, order: Order, guard: mmu::VmJobGuard, - aggregate: &Arc, + aggregate: &super::fence::Aggregate, inputs: &[file::SyncItem], destinations: Destinations, ) -> Result { @@ -189,22 +208,62 @@ impl Queue { }; let index = engine.index(); if let Validated::Render { ray_query, .. } = &command { - let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; - if blocks > self.context.render_blocks() { return Err(EINVAL); } + let blocks = self + .backend + .render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { + return Err(EINVAL); + } } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; + let Validated::Compute { scratch, .. } = &command else { + return Err(EINVAL); + }; self.ensure_compute(scratch.pages())?; } + // Only convert inputs when no independent host-prefix wait could keep + // the retained source bookkeeping queued behind an external signal. + if engine == Engine::Compute { + super::trace::inputs(self.context.id(), order.sequence, 0, inputs.len(), None); + } + let sources = if engine == Engine::Compute + && !inputs.is_empty() + && !destinations.has_outputs() + && self.frontiers[0].iter().all(|(sequence, fences)| { + !Order::contains(order.wait_through[0], *sequence) || fences.status() > 0 + }) + && self.frontiers[1].iter().all(|(sequence, fences)| { + !Order::contains(order.wait_through[1], *sequence) + || fences.ready.raw() != fences.completed.raw() + || fences.status() > 0 + }) + && self.failures.iter_mut().enumerate().all(|(index, failure)| { + failure.earliest().is_none_or(|(sequence, _)| { + !Order::contains(order.wait_through[index], sequence) + }) + }) { + self.backend.input_sources(inputs) + } else { + None + }; + if engine == Engine::Compute { + super::trace::inputs( + self.context.id(), + order.sequence, + 1, + inputs.len(), + sources.as_deref(), + ); + } + // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. let host_predecessors = self.frontiers[index] .iter() .take_while(|(sequence, _)| { - engine == Engine::Render - && Order::contains(order.wait_through[index], *sequence) + engine == Engine::Render && Order::contains(order.wait_through[index], *sequence) }) .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( @@ -216,9 +275,12 @@ impl Queue { destinations, engine == Engine::Compute, host_predecessors.clone().next().is_some(), - if engine == Engine::Render { self.backend.feed() } else { None }, + self.backend.feed(), + sources.clone(), + self.usage.clone(), )?; - let publication = completion.publication() + let publication = completion + .publication() .map(|target| PreparedPublication::new(target, host_predecessors)) .transpose()?; if engine == Engine::Compute { @@ -249,14 +311,17 @@ impl Queue { .iter() .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) .count(); - let failures: [Option<&Fence>; 2] = core::array::from_fn(|index| { - self.failures[index].as_ref().and_then(|(sequence, fence)| { - Order::contains(order.wait_through[index], *sequence).then_some(fence) - }) - }); + let mut failures: [Option; 2] = [None, None]; + for (index, failure) in self.failures.iter_mut().enumerate() { + failures[index] = failure.earliest().and_then(|(sequence, fences)| { + Order::contains(order.wait_through[index], sequence).then(|| fences.completed.clone()) + }); + } let failure_count = failures.iter().flatten().count(); - let mut dependencies = - KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; + let mut dependencies = KVec::with_capacity( + inputs.len() + count + timestamp_count + failure_count, + GFP_KERNEL, + )?; let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; @@ -277,7 +342,7 @@ impl Queue { // Successful completion includes host timestamp publication. Its // ready fence adds no ordering and would prevent shallow inline // publication. Failed and pending producers still need checking. - if job::fence_status(&fences.completed) > 0 { + if fences.status() > 0 { continue; } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; @@ -294,13 +359,26 @@ impl Queue { if job::fence_status(fence) > 0 { continue; } + if sources + .as_ref() + .is_some_and(|sources| sources.contains_input(fence)) + { + continue; + } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } - let early = engine == Engine::Render - && dependencies.is_empty() - && self.frontiers[index].len() < EARLY_DEPTH; + if let Some(sources) = &sources { + for ready in sources.publication_waits() { + dependencies.push(ready.clone(), GFP_KERNEL)?; + checked_inputs.push(ready.clone(), GFP_KERNEL)?; + } + } + // Publish from the ioctl whenever nothing has to wait, at any depth. + // The backend falls back to the scheduler behind any backlog or on any + // capacity miss, so submission order is preserved. + let early = dependencies.is_empty(); let packet = Packet::new( id, self.context.clone(), @@ -314,10 +392,11 @@ impl Queue { Engine::Render => &mut self.render, Engine::Compute => &mut self.compute, }; - let fences = scheduler - .as_mut() - .ok_or(ENODEV)? - .enqueue(packet, dependencies, early, publication)?; + let fences = + scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) @@ -384,8 +463,10 @@ impl Queue { impl crate::queue::Queue for Queue { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), - super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; + let pages = [ + super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages(), + ]; let blocks = self.backend.render_scs_blocks(pages)?; self.context.reserve_render_blocks(blocks) } @@ -399,22 +480,28 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new( - &self.fences, - syncs, - input_count, - &self.allocation_failure, - )?; + let mut outputs = Outputs::new(&self.fences, syncs, input_count, &self.allocation_failure)?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } let (mut pending, guards) = self.prepare_submission(bytes, objects)?; + if pending + .iter() + .any(|(command, _, _)| matches!(command, Validated::Render { .. })) + { + self.context.note_render(); + } if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { - for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } + for (command, _, _) in pending.iter_mut() { + command.allow_long_job(); + } } let mut guards = guards.into_iter(); - let aggregate = outputs.publish(self.context.clone())?; + let firmware_source = pending.len() == 1 + && matches!(&pending[0].0, Validated::Compute { .. }) + && !pending[0].2.has_outputs(); + let aggregate = outputs.publish(self.context.clone(), firmware_source)?; for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( @@ -429,6 +516,8 @@ impl crate::queue::Queue for Queue { } Ok(()) })(); + // Everything this ioctl publishes is visible; release what it pruned. + self.finished.clear(); if let Err(error) = result { self.context.status().record_if_device_loss(error); if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index fb6c77fbdf04f..2436cfd33ed35 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -7,7 +7,7 @@ pub(crate) use observation::Observed; mod dependencies; mod memory; pub(crate) mod parked; -mod publication; +pub(super) mod publication; pub(crate) mod retirement; mod warm; @@ -46,6 +46,9 @@ pub(crate) struct RenderDependencies { pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, + /// Fragment kicks of other pairs' overlapping same-VM renders. + pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], + pub(crate) hazard_count: usize, } impl RenderDependencies { @@ -55,6 +58,8 @@ impl RenderDependencies { tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, + hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], + hazard_count: 0, }; pub(crate) fn render(&self) -> &[KickDependency] { &self.render[..self.render_count] @@ -78,12 +83,13 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - fn qos_publish( + /// Publishes `owners` in order with one pass over the QoS table. + fn qos_publish_pair( &mut self, - owner: qos::Owner, + owners: [qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result; + ) -> Result<[qos::Publication; 2]>; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; @@ -136,6 +142,8 @@ struct Active { drained_result: Option>, /// First observation of this group retired and unlinked without execution. abandoned_since: Option, + /// First observation of this group executed and consumed while its stamps lagged. + stamp_wait_since: Option, } /// A firmware-retired, unexecuted group must hold on two observations this far @@ -186,6 +194,9 @@ pub(crate) struct Pair { /// An owner change in flight: the previous scheduler page and the ordinal of /// the new owner's first command, until that command leaves the pair. switch: Option<(u64, u64)>, + /// Completion values of the two kicks of the last publication, kept with the configuration + /// cache: what each queue holds once that publication completed. + published_completions: Option<[u64; 2]>, } impl Pair { #[inline(never)] @@ -237,8 +248,7 @@ impl Pair { let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; - let (fragment, fragment_alias) = - kick::Queue::render(alloc, ids[1], context.vm(), rings)?; + let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm(), rings)?; let (memory, aliases) = memory::Memory::new( alloc, pool, @@ -284,6 +294,7 @@ impl Pair { context_update: true, executed: 0, switch: None, + published_completions: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -324,7 +335,11 @@ impl Pair { self.replay.front().cloned() } pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self.replay.front().is_some_and(|front| Arc::ptr_eq(front, packet)) { + if !self + .replay + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { return Err(EIO); } self.replay.pop(); @@ -351,14 +366,20 @@ impl Pair { addresses: RenderAddresses, ) -> Result { if !self.room() - || self.replay.front().is_some_and(|front| !Arc::ptr_eq(front, packet)) + || self + .replay + .front() + .is_some_and(|front| !Arc::ptr_eq(front, packet)) || !Arc::ptr_eq(&self.memory.context, &packet.context) || !packet.context.is_current() { return Err(EBUSY); } let Validated::Render { - pass, attachments, ray_query, .. + pass, + attachments, + ray_query, + .. } = &packet.command else { return Err(EINVAL); @@ -368,6 +389,7 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; + self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); @@ -452,11 +474,9 @@ impl Pair { true => attachments[1].as_slice(), false => &[][..], }; - let result = self.memory.write_descriptors( - &args, - attachments, - packet.context.id().try_into()?, - ); + let result = + self.memory + .write_descriptors(&args, attachments, packet.context.id().try_into()?); let mcache = match result { Ok(mcache) => mcache, Err(error) => { @@ -484,7 +504,7 @@ impl Pair { struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, - fragment: [KickDependency; 3], + fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], fragment_count: usize, } @@ -495,7 +515,7 @@ fn render_barriers( ) -> RenderBarriers { let mut tiling_barriers = [KickDependency::ZERO; 3]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; let mut fragment_count = 1; for dependency in dependencies.render() { if fragment_stage { @@ -521,6 +541,11 @@ fn render_barriers( tiling_barriers[tiling_count] = compute; tiling_count += 1; } + // Only tile stores overlap: the tiling kick stays concurrent. + for hazard in &dependencies.hazards[..dependencies.hazard_count] { + fragment_barriers[fragment_count] = *hazard; + fragment_count += 1; + } RenderBarriers { tiling: tiling_barriers, tiling_count, @@ -571,12 +596,16 @@ impl Pair { let stamps = host.stamps(qids)?; fence(Ordering::Acquire); let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); - self.memory - .completion_links(ordinal, kicks, payload, !covered[1])?; + self.memory.completion_links( + ordinal, + kicks, + payload, + self.variant.fragment_busy && !covered[1], + )?; let leads = match self.variant.lead_commands { true => { let init_manager = !self.memory.manager.initialized(); - self.memory.write_leads(ordinal, kicks[0], init_manager)? + self.memory.write_leads(ordinal, init_manager)? } false => [false; 2], }; @@ -637,10 +666,28 @@ impl Pair { } else { [None; 2] }; - let present = [ - publication::required(cache[0], configs[0]), - publication::required(cache[1], configs[1]), - ]; + // A covered queue holds the completion value of its newest kick. When that kick is the + // previous publication of this pair in this recovery epoch, the seed would reload the + // value already held, so the configuration record is needed only for its identity. + // Each record costs the firmware time before it starts the kick. + let held = self + .published_completions + .take() + .filter(|_| self.recovery_generation == epoch); + let present = [0, 1].map(|stage| { + let mut config = configs[stage]; + let reload = kicks[stage] + .prev() + .completion_value(payload.wrapping_sub(1)); + if config.completion_seed_valid != 0 + && self.installed[stage] + && held.is_some_and(|held| held[stage] == reload) + { + config.completion_seed_valid = 0; + config.completion_seed = 0; + } + publication::required(cache[stage], config) + }); let cursors = self.memory.graph.cursors()?; if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); @@ -649,7 +696,10 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let Validated::Render { pass, ray_query, .. } = &packet.command else { + let Validated::Render { + pass, ray_query, .. + } = &packet.command + else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; @@ -752,10 +802,7 @@ impl Pair { windows[0].target().try_into()?, self.outer_started & ring == 0, )?; - let event_slot = memory - .graph - .support - .word(abi::storage::EVENT_SLOTS + 4)?; + let event_slot = memory.graph.support.word(abi::storage::EVENT_SLOTS + 4)?; let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); let inner = memory.graph.support.word(0x28000)?; let next_inner = (next_ordinal as u32).wrapping_mul(2); @@ -780,6 +827,7 @@ impl Pair { failed: false, drained_result: None, abandoned_since: None, + stamp_wait_since: None, }) .map_err(|_| EBUSY)?; self.prepared = None; @@ -790,10 +838,12 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - qos_publications[1] = - Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); - qos_publications[0] = - Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); + let [fragment, tiling] = host.qos_publish_pair( + [qos_owners[1], qos_owners[0]], + context.scheduler_va(), + policy, + )?; + qos_publications = [Some(tiling), Some(fragment)]; // A class change happens only at a rebind, after the pair drained. if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; @@ -840,6 +890,9 @@ impl Pair { } }; self.ordinal = next_ordinal; + if let Some(fragment) = KickDependency::new(qids[1], kicks[1]) { + packet.completion.note_render_publication(fragment); + } self.cursors = windows.map(|window| window.target()); for stage in 0..2 { if present[stage] { @@ -848,6 +901,7 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; + self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); @@ -903,7 +957,11 @@ impl Pair { let policy = packet.context.policy(); // A registration of another class can only follow a rebind, which // requires the pair to have drained. - if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + if self + .registered + .is_some_and(|class| class != policy.priority()) + && self.in_flight() + { return Err(EIO); } Ok((epoch, prepared.mcache, policy)) @@ -941,6 +999,12 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; + crate::g17::trace::publish( + crate::g17::trace::RENDER, + self.qids()[1], + self.ordinal as u32, + packet, + ); host.note_submission() })(); if let Err(error) = publish { @@ -993,12 +1057,124 @@ impl Pair { }, ] } + /// Bounded first-fault evidence while the recovery owner has firmware + /// halted, before it acknowledges state 1. Never changes retirement. + pub(crate) fn report_recovery(&self, stamps: Result<[u32; 2]>) { + pr_warn!("G17: recovery render slot={} qids={:?} active={} prepared={} quarantined={} stamps={:?} freelist={:?}\n", + self.slot, self.qids(), self.active.len(), self.prepared.is_some(), + self.quarantined, stamps, self.free_list.counters()); + self.memory.manager.report_recovery(); + for active in self.active.iter().take(4) { + if let Validated::Render { pass, .. } = &active.packet.command { + pr_warn!("G17: recovery render geometry ordinal={} width={} height={} layers={} layout={:?}\n", + active.ticket.ordinal, pass.width, pass.height, pass.layers, + target::Layout::new(pass, self.clusters, self.variant.programs)); + } + pr_warn!( + "G17: recovery render ordinal={} published={} ctx={} seq={} snapshot={:?}\n", + active.ticket.ordinal, + active.outer_published, + active.packet.context.id(), + active.packet.order.sequence, + observation::snapshot(&self.memory, active.ticket.ordinal) + ); + } + } + + /// Latches when the front group executed and the firmware consumed its ring entries while + /// `stamps` still lag it, and reports whether that has held for + /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it + /// on its pass record instead of waiting for stamps no later event will move. + pub(crate) fn note_stamp_wait( + &mut self, + stamps: [u32; 2], + state: u32, + now: u64, + ) -> Result { + if self.quarantined || self.prepared.is_some() || state != 0 { + return Ok(false); + } + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + if stamps + .into_iter() + .all(|stamp| retirement::stamp_covers(stamp, published)) + { + front.stamp_wait_since = None; + return Ok(false); + } + let snapshot = observation::snapshot(&self.memory, front.ticket.ordinal)?; + if !retirement::observe(&front.ticket, snapshot).is_ok_and(|seen| seen.complete) { + front.stamp_wait_since = None; + return Ok(false); + } + let since = *front.stamp_wait_since.get_or_insert(now); + Ok(Self::stamp_wait_expired(front, since, now)) + } + + /// Whether the front's stamp wait latched at `since` has run out by `now`. While other kicks + /// of the group's ioctl still run, its stamps are not late: the wait restarts and the pair + /// stays latched for the rescan. + fn stamp_wait_expired(front: &mut Active, since: u64, now: u64) -> bool { + let drained = || { + // A node that cannot be read keeps the bound from the latch. + front + .packet + .completion + .work_state() + .and_then(|work| work.drained()) + .unwrap_or(true) + }; + match retirement::stamp_wait(since, now, drained) { + retirement::StampWait::Waiting => false, + retirement::StampWait::Restart(now) => { + front.stamp_wait_since = Some(now); + false + } + retirement::StampWait::Expired => true, + } + } + + /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair + /// without a completion notification, which may never come for such a group. + pub(crate) fn stamp_waiting(&self) -> bool { + self.active + .front() + .is_some_and(|front| front.stamp_wait_since.is_some()) + } + + /// The watchdog of `packet` expired while it is this pair's front: if the firmware executed + /// it and consumed its ring entries while the stamps lag, it retires on its pass record now + /// instead of failing. Failing would give up on the stamps as well. + pub(crate) fn expire_stamp_wait( + &mut self, + packet: &Arc, + stamps: [u32; 2], + state: u32, + ) -> Result { + if !self + .active + .front() + .is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) + { + return Ok(false); + } + if !self.note_stamp_wait(stamps, state, crate::g17::now_ns())? && !self.stamp_waiting() { + return Ok(false); + } + if let Some(front) = self.active.front_mut() { + front.stamp_wait_since = Some(0); + } + Ok(true) + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { - masks.is_none_or(|masks| { - self.qids() - .into_iter() - .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) - }) + masks.is_none_or(|masks| completion_noted(self.qids(), masks)) } /// The oldest published group still running, for cross-VM timeout attribution /// (crate::g17::timeout). A quarantined pair's work holds the GPU until the firmware drains @@ -1048,7 +1224,9 @@ impl Pair { return Err(EINVAL); }; let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; - self.memory.manager.note_required(required, crate::g17::now_ns()); + self.memory + .manager + .note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); } @@ -1073,6 +1251,28 @@ impl Pair { .front() .map(|active| active.packet.completion.spared()) } + /// A recovery record names one of this pair's queues and the completion + /// stamp of the group it interrupted, `(ordinal + 1) << 8` like the pair's + /// stamps. Without a stamp match the oldest published group is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self + .active + .iter() + .filter(|active| active.outer_published) + .find(|active| (active.ticket.ordinal.wrapping_add(1) as u32) & 0xff_ffff == sequence) + { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.outer_published) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { for active in self.active.iter() { visit(active.packet.completion.status()); @@ -1089,7 +1289,10 @@ impl Pair { /// A slot stamp must name retained work before its execution markers are meaningful. pub(crate) fn pass_started(&self, stamp: u64) -> Option { const MASK: u64 = (1 << 40) - 1; - let ordinal = self.active.iter().map(|active| active.ticket.ordinal) + let ordinal = self + .active + .iter() + .map(|active| active.ticket.ordinal) .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) } @@ -1101,6 +1304,7 @@ impl Pair { pub(crate) fn resync(&mut self) { self.configs = [None; 2]; + self.published_completions = None; for kick in &mut self.kicks { kick.clear_parent_after_recovery(); } @@ -1120,7 +1324,11 @@ impl Pair { self.quarantined = true; // Successors published after recovery can still fail behind its spared // head. Classify the whole failing group before any fence can settle. - if self.active.iter().any(|active| active.packet.completion.spared()) { + if self + .active + .iter() + .any(|active| active.packet.completion.spared()) + { for active in self.active.iter() { active.packet.completion.classify(recovery::Class::Spared); } @@ -1187,6 +1395,9 @@ impl Pair { { return Err(EIO); } + let stampless = front + .stamp_wait_since + .is_some_and(|since| Self::stamp_wait_expired(front, since, crate::g17::now_ns())); let mut observer = Observer { memory: &self.memory, qids, @@ -1196,7 +1407,13 @@ impl Pair { if observed.ticket != front.ticket { return Err(EIO); } - retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + if stampless { + retirement::poll_stampless(&mut front.ticket, &mut observer) + } else { + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } + } else if stampless { + retirement::poll_stampless(&mut front.ticket, &mut observer) } else { retirement::poll(&mut front.ticket, &mut observer) }; @@ -1211,6 +1428,9 @@ impl Pair { }; let ordinal = front.ticket.ordinal; let context_update = front.context_update; + if stampless { + retirement::note_stampless(u16::from(self.slot), ordinal); + } self.note_retirement(ordinal, context_update); let front = self.active.front_mut().ok_or(EIO)?; if !front.flist_completed { @@ -1223,6 +1443,14 @@ impl Pair { front.qos_completed = true; front.flist_completed = true; } + crate::g17::recovery::note_retired(); + crate::g17::trace::complete( + crate::g17::trace::RENDER, + qids[1], + ordinal as u32, + &front.packet, + [timestamps[0], timestamps[3]], + ); defer(Deferred::Retired( front.packet.completion.clone(), Ok(timestamps), @@ -1254,6 +1482,9 @@ impl Pair { if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { return Ok(false); } + if let Some(progressed) = self.settle_interrupted_front(host, defer, grace)? { + return Ok(progressed); + } use crate::g17::replay::{self, Decision, Group}; let qids = self.qids(); let memory = &self.memory; @@ -1264,10 +1495,9 @@ impl Pair { let blamed = active.packet.completion.blamed(); let exhausted = active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; - // decide() stops at the first group that is not abandoned, an error included. - let abandoned = innocent - && !blamed - && (|| -> Result { + // decide() stops at the first group that is still live, an error included. + let (abandoned, ran, lost) = if innocent && !blamed { + (|| -> Result<(bool, bool, bool)> { let mut observer = Observer { memory, qids, @@ -1278,20 +1508,43 @@ impl Pair { let stamps = observer.stamps()?; let (_, state) = observer.host.epoch()?; // A torn sample is no proof; a later poll or the timeout looks again. - Ok(matches!( - retirement::abandoned(&active.ticket, snapshot, stamps, state, false), - Ok(true) + let witness = |executed| { + matches!( + retirement::abandoned( + &active.ticket, + snapshot, + stamps, + state, + executed + ), + Ok(true) + ) + }; + let unexecuted = witness(false); + // Interrupted work is run again only if it never started, and work that + // another queue waited on never: the wait ended when firmware retired it. + let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) + || active.packet.completion.has_firmware_consumer(); + Ok(( + unexecuted && !lost, + !unexecuted && witness(true), + unexecuted && lost, )) })() .unwrap_or_else(|error: Error| { failure = Some(error); - false - }); + (false, false, false) + }) + } else { + (false, false, false) + }; Group { innocent, blamed, abandoned, exhausted, + ran, + lost, } }); let decision = replay::decide(groups); @@ -1306,7 +1559,7 @@ impl Pair { front.abandoned_since = None; return Ok(false); } - Decision::Replay => {} + Decision::Replay | Decision::Fail => {} } if grace { let Some(since) = front.abandoned_since else { @@ -1317,6 +1570,9 @@ impl Pair { return Ok(false); } } + if decision == Decision::Fail { + return self.fail_dropped(host, defer); + } // decide() lets only the front be exhausted. An exhausted front fails, and its // successors, which consume its output, fail with it rather than run without it. let mut failing = self.active.front().is_some_and(|front| { @@ -1369,11 +1625,126 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; packet.reset_publication_for_replay(); + self.published_completions = None; self.replay.push(packet).map_err(|_| EIO)?; } Ok(true) } + /// Firmware retired every group of the pair after a restart, but some were dropped + /// unexecuted and cannot run again in order (see `replay::decide`). Every group fails now, + /// the ones that ran too: they consumed output the dropped ones never wrote. The work is + /// innocent, so it fails without poisoning its VM, with ENODATA, which its owner sees, + /// instead of waiting for the render timeout to fail it with ETIMEDOUT. + fn fail_dropped( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + pr_warn!( + "G17: render slot {}: a GPU recovery dropped groups that cannot run again in order; failing {} groups\n", + self.slot, + self.active.len() + ); + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + while let Some(front) = self.active.front_mut() { + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let completion = front.packet.completion.clone(); + completion.classify(recovery::Class::Spared); + defer(Deferred::Retired(completion, Err(ENODATA)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + Ok(true) + } + + /// A front group a restart interrupted after its tiler started, which firmware + /// then retired without writing its fragment end, may have run partly. It is + /// not replayed (blending or storage writes could apply twice) and fails now, + /// reported to its owner, instead of waiting out the render timeout. Returns + /// None when the front is not such a group. + fn settle_interrupted_front( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + grace: bool, + ) -> Result> { + use retirement::Memory; + let qids = self.qids(); + let Some(front) = self.active.front() else { + return Ok(None); + }; + if !front.outer_published || front.failed || !front.packet.completion.interrupted() { + return Ok(None); + } + let ordinal = front.ticket.ordinal; + let ticket = front.ticket; + let mut observer = Observer { + memory: &self.memory, + qids, + host: &mut *host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if snapshot.untouched() + || !matches!( + retirement::abandoned(&ticket, snapshot, stamps, state, false), + Ok(true) + ) + { + return Ok(None); + } + let now = crate::g17::now_ns(); + let front = self.active.front_mut().ok_or(EIO)?; + if grace { + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(Some(false)); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(Some(false)); + } + } + if !front.flist_completed { + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + pr_warn!( + "G17: render slot {}: group {} was interrupted by a GPU recovery after it started; failing it\n", + self.slot, + ordinal + ); + let packet = self.active.front().ok_or(EIO)?.packet.clone(); + defer(Deferred::Retired(packet.completion.clone(), Err(ECANCELED)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + Ok(Some(true)) + } + /// The pending replay cannot proceed (closed owner, cancellation or a /// publication refusal). Its successors cannot run without it and fail with /// it. ECANCELED leaves the VM usable; any other error is the ordinary @@ -1415,7 +1786,44 @@ impl Pair { let counters = self.free_list.counters()?; let stamps = host.stamps(self.qids())?; let (_, state) = host.epoch()?; - Ok(retirement::drained(self.ordinal, stamps, counters, state)) + Ok(retirement::drained(self.ordinal, stamps, counters, state) + || (state == 0 && self.executed_unstamped() && self.context_killed())) + } + + /// The firmware acknowledged a kill of this pair's context (scheduler page): it no longer + /// walks that context's work list, so the commands it never unlinked can be released. + fn context_killed(&self) -> bool { + crate::g17::queue::compute::exit::State::read(&self.memory.context) + .is_ok_and(|state| state.killed != 0) + } + + /// The pair's context may be killed although its commands were not retired: they all ran + /// (`executed_unstamped`), so the kill leaves nothing queued for the firmware to run. + /// Its memory stays until the kill is acknowledged. + pub(crate) fn executed_for_kill(&self, context: &Arc) -> bool { + Arc::ptr_eq(context, &self.memory.context) && self.executed_unstamped() + } + + /// Every command of this quarantined pair ran, its ring entries were consumed and its + /// free-list blocks returned, yet the firmware never stores the stamp of the last one. + /// On Neo this is the render work of a context whose compute command failed: it runs + /// about a second after the failure, but its stamp never arrives and it stays linked in + /// the firmware's job list, so neither the stamp proof nor the abandoned-group proof + /// ever holds and the pair would stay quarantined, keeping its context, until the + /// processors stop. + fn executed_unstamped(&self) -> bool { + self.quarantined + // Only for a closed owner: a live owner would go on publishing into a context + // whose work list still links the unstamped node. + && self.owner.is_none() + && self.prepared.is_none() + && self.active.len() != 0 + && self.free_list.counters().is_ok_and(|counters| counters[0] == counters[1]) + && self.active.iter().all(|active| { + active.outer_published + && observation::snapshot(&self.memory, active.ticket.ordinal) + .is_ok_and(retirement::Snapshot::executed_and_consumed) + }) } /// Account one witnessed tracker, preserving all completion ownership. @@ -1429,6 +1837,10 @@ impl Pair { let qids = self.qids(); let qos = self.qos_owners(); let buffer = self.buffer_id(); + // Commands the firmware never unlinked are released only after it acknowledged the + // kill of their context, on every path (see `firmware_drained`). + let unstamped = self.executed_unstamped(); + let killed = unstamped && self.context_killed(); let Some(active) = self.active.iter_mut().find(|a| !a.flist_completed) else { return Ok(false); }; @@ -1439,6 +1851,12 @@ impl Pair { }; let result = match retirement::poll(&mut active.ticket, &mut observer)? { retirement::Poll::Complete(ticks) => Ok(ticks), + // Its commands ran without a stamp (see `executed_unstamped`): they already failed + // with the quarantine, and the client sees them cancelled. + retirement::Poll::Pending | retirement::Poll::Retry if killed => { + crate::g17::trace::teardown(2, u64::from(self.slot), active.ticket.ordinal, 0); + Err(ECANCELED) + } retirement::Poll::Pending | retirement::Poll::Retry => { core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); let snapshot = observer.snapshot(active.ticket.ordinal)?; @@ -1515,15 +1933,13 @@ impl Pair { // own even with successors queued, so the pair is not quarantined into a sticky EIO. It // fails with ECANCELED, which spared work never reports as device loss; consecutive // abandoned spared successors are retired the same way. - let front_spared = self.active.front().is_some_and(|a| { - Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared() - }); + let front_spared = self + .active + .front() + .is_some_and(|a| Arc::ptr_eq(&a.packet, packet) && a.packet.completion.spared()); if !self.quarantined && (self.active.len() == 1 || front_spared) { let mut retired = false; - loop { - let Some(active) = self.active.front() else { - break; - }; + while let Some(active) = self.active.front() { let spared = active.packet.completion.spared(); if retired && !spared { break; @@ -1770,6 +2186,71 @@ impl Pair { } /// The scheduler page these queues last ran a command for, when the next /// publication for `context` changes the queues' owner. + /// The steady state in which a command may publish without the device + /// mutex: the graph is installed and registered at `priority`, it last ran + /// `context`'s scheduler, and nothing is replayed, settled, aliased or + /// switched on it. + pub(crate) fn direct_ready(&self, context: &Context, priority: u8) -> bool { + self.graph_started + && self.installed == [true; 2] + && self.registered == Some(priority) + && priority < 8 + && self.outer_started & (1u8 << priority) != 0 + && self.executed == context.scheduler_va() + && self.switch.is_none() + && self.owner.is_some() + && !self.quarantined + && self.terminal.is_none() + && self.pending_aliases.is_none() + && self.kick_aliases.is_none() + && self.prepared.is_none() + && self.replay.len() == 0 + && self.displaced.is_none() + && self.active.room() + } + /// The render prefix of `packet` on this pair, as the registry collects it + /// for an owner whose only pair this is. + pub(crate) fn own_dependencies( + &self, + owner: u64, + packet: &Arc, + ) -> Result { + let mut dependencies = RenderDependencies::EMPTY; + let mut frontier = crate::g17::dependency::Frontier::new(); + self.collect_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + false, + )?; + for dependency in frontier.as_slice() { + *dependencies + .render + .get_mut(dependencies.render_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.render_count += 1; + } + if dependencies.render_count != 0 + && matches!(&packet.command, Validated::Render { pass, .. } if pass.fragment_barriers) + { + let mut tiling = crate::g17::dependency::Frontier::new(); + self.collect_tiling_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut tiling, + )?; + for dependency in tiling.as_slice() { + *dependencies + .tiling + .get_mut(dependencies.tiling_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.tiling_count += 1; + } + } + Ok(dependencies) + } pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) } @@ -1777,8 +2258,12 @@ impl Pair { /// not left the pair yet. pub(crate) fn switch_in_flight(&self) -> Option { let (from, first) = self.switch?; - (self.replay.len() != 0 || self.active.iter().any(|active| active.ticket.ordinal == first)) - .then_some(from) + (self.replay.len() != 0 + || self + .active + .iter() + .any(|active| active.ticket.ordinal == first)) + .then_some(from) } pub(crate) fn take_retired_binding(&mut self) -> Option { if self.displaced.is_some() { @@ -1943,7 +2428,9 @@ impl Pair { self.memory.context = binding.context.clone(); self.owner = Some(binding.owner); self.initial_packet = binding.initial_packet; - if foreign { self.graph_started = false; } + if foreign { + self.graph_started = false; + } self.context_update = true; if foreign { self.cursors = [0; 2]; @@ -1952,3 +2439,45 @@ impl Pair { Ok(()) } } + +/// Whether `masks`, the stamp notifications collected for a pair whose tiling and fragment +/// queues are `qids`, can mean a finished pass: the fragment queue's bit. A fragment kick starts +/// only after its fragment barrier saw the tiling stamp, and retirement still checks both stamps +/// in memory, so the tiling notification is not needed to retire. +pub(crate) fn completion_noted(qids: [u8; 2], masks: [u64; 2]) -> bool { + let fragment = usize::from(qids[1]); + masks[fragment / 64] & (1u64 << (fragment % 64)) != 0 +} + +#[kunit_tests(asahi_g17_render_completion_noted)] +mod completion_noted_tests { + use super::completion_noted; + + fn bit(qid: u8) -> [u64; 2] { + let mut masks = [0; 2]; + masks[usize::from(qid) / 64] |= 1u64 << (qid % 64); + masks + } + + #[test] + fn fragment_notification_alone_selects() { + assert!(completion_noted([3, 4], bit(4))); + assert!(completion_noted([70, 100], bit(100))); + } + + #[test] + fn tiling_notification_alone_does_not_select() { + assert!(!completion_noted([3, 4], bit(3))); + assert!(!completion_noted([70, 100], bit(70))); + assert!(!completion_noted([3, 4], [0; 2])); + } + + #[test] + fn other_queues_do_not_select() { + assert!(!completion_noted([3, 4], bit(5))); + assert!(!completion_noted([3, 68], bit(4))); + let mut both = bit(3); + both[1] |= bit(68)[1]; + assert!(completion_noted([3, 68], both)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 80ee0b871cc82..90a11ca977684 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,8 +31,8 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; -/// The lead command of a stage (InitBM before tiling, the barrier before fragment), after the -/// stage's configuration record in its slot ([`crate::g17::fw::Variant::lead_commands`]). +/// The lead command of a stage (InitBM before tiling), after the stage's configuration record +/// in its slot ([`crate::g17::fw::Variant::lead_commands`]). pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. @@ -528,15 +528,44 @@ impl Memory { } const _: () = { + // Every unretired render of a pair holds its own command record. + assert!(DEPTH <= RECORDS); assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); assert!(size_of::() <= LEAD); - assert!(LEAD + size_of::() <= CONFIG_STRIDE); assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; impl Memory { + /// Requests the tiling descriptor slot and both queue configuration records of + /// `ordinal` for writing. The firmware accessed them for the command that used them + /// last, so the host no longer holds their lines; requesting them before the rest of + /// the preparation overlaps those transfers with it instead of stalling the writes one + /// line at a time. + pub(super) fn prefetch_descriptors(&self, ordinal: u64) { + let record = ordinal as usize % RECORDS; + let config = size_of::(); + let ranges = [ + (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), + (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), + (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), + ]; + for (object, offset, size) in ranges { + let Ok(base) = object.pointer(offset, size) else { + continue; + }; + for line in (0..size).step_by(64) { + // SAFETY: A prefetch hint inside the retained mapping; it has no + // architectural effect on memory. + unsafe { + core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(line), + options(nostack, preserves_flags, readonly)); + } + } + } + } + pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, @@ -571,21 +600,10 @@ impl Memory { (&mut *tiling).write(args)?; let fragment = &mut *fragment; fragment.write(args)?; - self.registers.bytes_mut().fill(0); - for (index, array) in fragment.descriptor.arrays.iter().enumerate() { - for (entry, value) in array - .entries - .get(..(array.size & 0xffff) as usize) - .ok_or(EINVAL)? - .iter() - .enumerate() - { - self.registers.write( - index * size_of::() - + entry * size_of::(), - *value, - )?; - } + // Kick entries bind the arrays embedded in this descriptor + // (register_bindings); the client register page is never read. + for array in &fragment.descriptor.arrays { + array.entries.get(..(array.size & 0xffff) as usize).ok_or(EINVAL)?; } fragment.set_mcache( args.fragment.descriptor_va, @@ -629,14 +647,8 @@ impl Memory { } /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the - /// tiling descriptor when `init_manager`, and the fragment barrier. Returns which stages - /// have one. - pub(super) fn write_leads( - &mut self, - ordinal: u64, - tiling_kick: fw::kick::KickTimestamp, - init_manager: bool, - ) -> Result<[bool; 2]> { + /// tiling descriptor when `init_manager`. Returns which stages have one. + pub(super) fn write_leads(&mut self, ordinal: u64, init_manager: bool) -> Result<[bool; 2]> { if !self.owns(ordinal) { return Err(EIO); } @@ -645,23 +657,10 @@ impl Memory { .descriptors .pointer(Graph::offset(0, ordinal), size_of::())? .cast::(); - let fragment = self - .graph - .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? - .cast::(); - // SAFETY: Both views were checked during preparation and the ordinal still owns them; - // only integer fields are read. - let (ta, fragment_id) = unsafe { (&*ta, (*fragment).descriptor.current_id) }; + // SAFETY: The view was checked during preparation and the ordinal still owns it; only + // integer fields are read. + let ta = unsafe { &*ta }; let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; - let barrier = queue::FragmentBarrier::new( - ta.stamp_va, - tiling_kick, - ta.qid, - ta.stamp_value, - fragment_id, - ); - self.graph.config.write(CONFIG[1] + slot, barrier)?; if init_manager { let init = queue::InitBufferManager::new( ta.context_id, @@ -671,7 +670,7 @@ impl Memory { ); self.graph.config.write(CONFIG[0] + slot, init)?; } - Ok([init_manager, true]) + Ok([init_manager, false]) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 209ce0b8176ec..81d4306eda28c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -252,6 +252,7 @@ impl ParkedPair { context_update: true, executed: self.executed, switch: None, + published_completions: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 9f4cb05b6bede..d65c40c7cfade 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -19,7 +19,7 @@ fn identity(mut config: QueueConfig) -> QueueConfig { config.qos_update = 0; config } -pub(super) fn required(previous: Option, config: QueueConfig) -> bool { +pub(in crate::g17::queue) fn required(previous: Option, config: QueueConfig) -> bool { config.install != 0 || config.context_update != 0 || config.completion_seed_valid != 0 @@ -41,6 +41,10 @@ pub(super) struct Lane<'a> { pub(super) window: Window, } impl<'a> Lane<'a> { + #[expect( + clippy::too_many_arguments, + reason = "The publication lane describes independent fields of the firmware wire record." + )] pub(super) fn new( graph: &'a memory::Graph, stage: usize, @@ -94,7 +98,9 @@ impl<'a> Lane<'a> { push(announce_va); Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, + producer: graph + .queues + .word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 24aeed6bcd0ec..4377bbfe3e311 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -122,6 +122,17 @@ impl Snapshot { fn executed(self) -> bool { self.pass[0] != 0 && self.pass[1] != 0 } + + /// Neither the tiler start nor the fragment end was written: nothing ran. + pub(crate) fn untouched(self) -> bool { + self.pass == [0, 0] + } + + /// The command ran (tiler start and fragment end written) and the firmware consumed both + /// queues' ring entries. + pub(crate) fn executed_and_consumed(self) -> bool { + self.executed() && self.cursors.iter().all(|cursors| cursors.done == cursors.write) + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -144,6 +155,85 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result bool) -> StampWait { + if now.saturating_sub(since) < STAMP_FALLBACK_NS { + StampWait::Waiting + } else if since == 0 || drained() { + StampWait::Expired + } else { + StampWait::Restart(now) + } +} + +static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + +/// Counts a render group that retired on its pass record because its completion stamps never +/// covered it; logs the first and every 64th. +pub(crate) fn note_stampless(slot: u16, ordinal: u64) { + let count = STAMPLESS.fetch_add(1, Ordering::Relaxed) + 1; + if count == 1 || count % 64 == 0 { + pr_info!( + "G17: lifetime total {} render groups retired on their pass record after {} ms without completion stamps (latest: slot {} ordinal {})\n", + count, + STAMP_FALLBACK_NS / 1_000_000, + slot, + ordinal + ); + } +} + +/// Retire an executed, consumed front group without its stamps, once [`STAMP_FALLBACK_NS`] +/// has passed. The caller decides that bound; this re-checks execution and consumption. Unlike +/// the stamped path, it has no post-maintenance evidence; see [`STAMP_FALLBACK_NS`]. +pub(crate) fn poll_stampless(ticket: &mut Ticket, memory: &mut impl Memory) -> Result { + fence(Ordering::Acquire); + let snapshot = memory.snapshot(ticket.ordinal)?; + let Ok(observation) = observe(ticket, snapshot) else { + return Ok(Poll::Retry); + }; + if !observation.complete { + return Ok(Poll::Pending); + } + let [start, end] = memory.pass(ticket.ordinal)?; + Ok(Poll::Complete([start, start, start, end])) +} + /// Outstanding publications span less than half the 24-bit stamp sequence. /// A zero stamp is valid when that sequence wraps. pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool { @@ -350,3 +440,94 @@ impl Tracker { .filter_map(Option::as_mut) } } + +#[kunit_tests(asahi_g17_render_retirement)] +mod executed_tests { + use super::*; + + fn snapshot(done: u32, write: u32, pass: [u64; 2]) -> Snapshot { + Snapshot { + cursors: [Cursors { done, read: done, write }; 2], + job_list_empty: false, + pass, + } + } + + #[test] + fn executed_and_consumed_needs_both_passes_and_consumed_rings() { + assert!(snapshot(3, 3, [1, 2]).executed_and_consumed()); + assert!(!snapshot(3, 3, [0, 0]).executed_and_consumed()); + assert!(!snapshot(3, 3, [1, 0]).executed_and_consumed()); + assert!(!snapshot(3, 3, [0, 2]).executed_and_consumed()); + assert!(!snapshot(2, 3, [1, 2]).executed_and_consumed()); + } + + /// The firmware state of a stalled pair as read on hardware: the front group executed and + /// its two ring entries per queue were consumed, but both stamps hold the previous kick. + struct Stalled { + snapshot: Snapshot, + stamps: [u32; 2], + } + impl Memory for Stalled { + fn snapshot(&mut self, _ordinal: u64) -> Result { + Ok(self.snapshot) + } + fn stamps(&mut self) -> Result<[u32; 2]> { + Ok(self.stamps) + } + fn pass(&mut self, _ordinal: u64) -> Result<[u64; 2]> { + Ok(self.snapshot.pass) + } + } + + fn stalled_ticket() -> Ticket { + let window = Window::new(1097, 2, 1280).unwrap(); + let previous = Cursors { done: 1097, read: 1097, write: 1097 }; + Ticket { ordinal: 406, windows: [window; 2], previous: [previous; 2] } + } + + fn stalled(pass: [u64; 2]) -> Stalled { + let cursors = Cursors { done: 1099, read: 1099, write: 1099 }; + Stalled { + snapshot: Snapshot { cursors: [cursors; 2], job_list_empty: false, pass }, + stamps: [406 << 8; 2], + } + } + + #[test] + fn lagging_stamps_keep_an_executed_group_pending() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 20]); + assert!(matches!(poll(&mut ticket, &mut memory), Ok(Poll::Pending))); + } + + #[test] + fn stampless_poll_retires_an_executed_consumed_group() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 20]); + assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Complete([10, 10, 10, 20])))); + } + + #[test] + fn stampless_poll_waits_for_the_fragment_end() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 0]); + assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending))); + } + + #[test] + fn stamps_wait_for_the_rest_of_the_ioctl() { + let since = 1_000_000; + let late = since + STAMP_FALLBACK_NS; + // The node is asked only once the bound passed. + assert_eq!(stamp_wait(since, late - 1, || panic!("node read early")), StampWait::Waiting); + // Other kicks of the group's ioctl still run: the bound restarts. + assert_eq!(stamp_wait(since, late, || false), StampWait::Restart(late)); + assert_eq!(stamp_wait(late, late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); + // A drained node with lagging stamps is the stall the bound exists for. + assert_eq!(stamp_wait(since, late, || true), StampWait::Expired); + // The watchdog's last resort does not wait for the node. + assert_eq!(stamp_wait(0, STAMP_FALLBACK_NS, || panic!("node read for an expired wait")), + StampWait::Expired); + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 89f8d33bdb80c..ca8da8b03545f 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -10,6 +10,7 @@ mod attribution; mod slot; mod safety; +pub(crate) use safety::{note_retired, since_retired}; pub(crate) use attribution::Sources; use attribution::Attribution; @@ -24,20 +25,34 @@ const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; const MAX_GUILTY_VMS: usize = 8; -const RECOVERY_POLLS: usize = 500; +/// One-millisecond ticks per handshake phase. The firmware finishes a restart only after the +/// work it interrupted has drained; with several clients running long ray-traversal jobs that +/// takes seconds, and giving up loses every client's device. Ten seconds matches the longest +/// execution allowance (`timeout::POLL_MS` plus renewals). +const RECOVERY_POLLS: usize = 10_000; -/// Per-QID classifications from the complete firmware recovery table. +/// Records in the firmware recovery table. +pub(super) const RECOVERY_RECORDS: usize = 256; + +/// Per-QID classifications from the firmware recovery table. /// Multiple records for a queue contribute all of their classification bits. #[derive(Default)] pub(super) struct Entries { guilty: u128, cancelled: u128, + /// Valid records, each naming one command that was in flight at the restart. + pub(super) listed: u32, + /// Listed commands found by their exact stamp, and by the oldest-work fallback. + pub(super) exact: u32, + pub(super) fallback: u32, } impl Entries { - pub(super) fn include(&mut self, flags: u32) { + /// Returns false at the first invalid record: the table is a packed list and + /// records after it are left over from an earlier restart. + pub(super) fn include(&mut self, flags: u32) -> bool { if flags & 1 == 0 { - return; + return false; } let qid = (flags >> 4) & 0x7f; let bit = 1u128 << qid; @@ -47,9 +62,22 @@ impl Entries { if flags & u32::from(CANCELLED) != 0 { self.cancelled |= bit; } + self.listed += 1; + true } } +/// How a listed recovery record was matched to retained work. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Interrupted { + /// No retained command of the queue. + None, + /// The command whose completion stamp equals the record's. + Exact, + /// No stamp matched; the queue's oldest unretired command was marked instead. + Oldest, +} + /// How an error completion affects its VM's sticky status and other jobs. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Class { @@ -71,6 +99,12 @@ pub(super) trait Queues { fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); /// Classify every in-flight completion; return QIDs of render slots with spared work. fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; + /// Mark the command a recovery record lists as interrupted. It may have + /// partly executed, so it must never report success; see + /// `Completion::note_interrupted`. + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> Interrupted; + /// Optional diagnostics only; absence cannot alter attribution or release. + fn report_blamed_render(&self, _qid: u8) {} fn settle_blamed_render(&mut self, qid: u8, error: Error); fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; @@ -85,7 +119,9 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - fn entries(&self) -> Result; + /// One recovery record's flags and completion stamp, read while firmware + /// is halted in state 3. + fn entry(&self, index: usize) -> Result<(u32, u32)>; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -119,7 +155,9 @@ impl State { restarts: safety::Storm::new(), } } - pub(super) fn timeout_limit(&mut self, now: u64) -> bool { self.timeouts.note(now) } + pub(super) fn timeout_limit(&mut self, now: u64) -> bool { + self.timeouts.note_unless_progress(now, safety::retired()) + } pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted @@ -192,6 +230,8 @@ pub(super) fn service( } } if !ready { + pr_warn!("G17: recovery gen {} did not reach state 1 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // Recheck ownership after the off-CPU waits, before any handshake mutation. @@ -203,6 +243,15 @@ pub(super) fn service( host.clear_timestamps()?; } let mut sources = host.fault_sources()?; + sources.report(request.generation, super::queue::compute::retirement::stampless_total()); + if let Some(qid) = request.blamed { + host.report_blamed_render(qid); + } + // The keys, monitor and pass-start evidence must describe the same halted + // instant. Firmware may update pass timestamps as soon as state 3 is + // acknowledged; sampling them during classification can blame a bystander. + let attribution = sources.attribute(request.blamed, + |qid, stamp| host.render_pass_started(qid, stamp)); let target_error = host.publish_recovery_targets(); host.transition(1, 2)?; let mut recovered = false; @@ -217,17 +266,31 @@ pub(super) fn service( host.wait_tick(); } if !recovered { + pr_warn!("G17: recovery gen {} did not reach state 3 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // A failed table read leaves only the explicit blamed QID as evidence of guilt. - let entries = host.entries().unwrap_or_default(); + // Every listed command was in flight when firmware stopped. Firmware may + // still retire it with stamps and records that look like success, so it is + // marked before firmware resumes and never reports success. + let entries = read_entries(host); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; - let (cancelled, blame) = classify(host, entries, request.blamed, &mut sources); + let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); + let (cancelled, blame) = classify(host, entries, request.blamed, attribution); + if listed != 0 { + pr_warn!( + "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", + listed, + exact, + fallback + ); + } if let Some(qid) = request.blamed { pr_warn!( "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", @@ -247,16 +310,35 @@ pub(super) fn service( Ok(ServiceResult { scan_error, target_error }) } +/// Walks the packed recovery table up to its first invalid record and marks +/// every listed command interrupted. A failed read keeps what was read so far. +fn read_entries(host: &mut H) -> Entries { + let mut entries = Entries::default(); + for index in 0..RECOVERY_RECORDS { + let Ok((flags, stamp)) = host.entry(index) else { + break; + }; + if !entries.include(flags) { + break; + } + match host.note_interrupted(((flags >> 4) & 0x7f) as u8, stamp) { + Interrupted::Exact => entries.exact += 1, + Interrupted::Oldest => entries.fallback += 1, + Interrupted::None => {} + } + } + entries +} + /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. /// Also returns whether the firmware's blamed QID is still guilty after attribution: a blamed /// render that had not started while others had (or with no culprit at all) is innocent, and its /// work is spared like that of every other innocent VM instead of failing with EIO. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> (u128, bool) { +fn classify(queues: &mut Q, entries: Entries, blamed: Option, attribution: Attribution) -> (u128, bool) { let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); let cancelled = entries.cancelled; - let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); match attribution { Attribution::Firmware => {} Attribution::Queues(qids) => guilty_qids = qids, @@ -316,3 +398,101 @@ fn classify(queues: &mut Q, entries: Entries, blamed: Option, sou }); (cancelled & !(guilty_qids & !collateral) & spared, blame) } + + +#[kernel::macros::kunit_tests(asahi_g17_recovery_snapshot)] +mod tests { + use super::*; + use core::cell::Cell; + + struct Halted { + recovery: State, + state: Cell, + reads: Cell, + known_keys: bool, + blamed: bool, + } + + impl Halted { + fn new(known_keys: bool) -> Self { + let mut recovery = State::new(); + assert!(recovery.request(0, 38).unwrap()); + Self { recovery, state: Cell::new(1), reads: Cell::new(0), + known_keys, blamed: false } + } + } + + impl Memory for Halted { + fn recovery_state(&self) -> Result { Ok(self.state.get()) } + fn host_recovery(&self) -> Result { Ok(0) } + fn fault_sources(&self) -> Result { + assert_eq!(self.state.get(), 1); + let mut sources = Sources::sample(3, |slot| { + Some((if self.known_keys { ((38 + slot as u64) << 40) | 1 } else { 0 }, 0)) + }); + sources.reason = Some(2); + Ok(sources) + } + fn transition(&self, expected: u32, next: u32) -> Result { + assert_eq!(self.state.get(), expected); + self.state.set(next); + Ok(()) + } + fn clear_timestamps(&self) -> Result { panic!("active render scheduler") } + fn entry(&self, _index: usize) -> Result<(u32, u32)> { + assert_eq!(self.state.get(), 3); + Ok((0, 0)) + } + fn wait_tick(&self) { + assert_eq!(self.state.get(), 2); + self.state.set(3); + } + } + + impl Queues for Halted { + fn render_scheduler_active(&self) -> bool { true } + fn render_pass_started(&self, qid: u8, _stamp: u64) -> Option { + // This is the regression oracle: state 0/2/3 is too late. A + // resumed peer may have started after the fault we are attributing. + assert_eq!(self.state.get(), 1); + self.reads.set(self.reads.get() + 1); + Some(qid == 39) + } + fn publish_recovery_targets(&self) -> Option { None } + fn oldest_spared(&self, _qid: u8) -> Option { None } + fn visit_owners(&self, _qid: u8, _visit: &mut dyn FnMut(&Arc)) {} + fn classify_work(&mut self, _class: &dyn Fn(&Arc) -> Class) -> u128 { 0 } + fn note_interrupted(&mut self, _qid: u8, _stamp: u32) -> Interrupted { Interrupted::None } + fn settle_blamed_render(&mut self, qid: u8, _error: Error) { + assert_eq!(qid, 38); + self.blamed = true; + } + fn settle_cancelled_renders(&mut self, _qids: u128) {} + fn rescan_compute(&mut self) -> Result { Ok(()) } + fn release_retired_compute(&mut self) {} + } + + impl Host for Halted { + fn recovery(&self) -> &State { &self.recovery } + fn recovery_mut(&mut self) -> &mut State { &mut self.recovery } + } + + #[test] + fn pass_evidence_is_frozen_before_the_first_acknowledgement() { + let mut host = Halted::new(true); + service(&mut host, || false).unwrap(); + assert_eq!(host.reads.get(), 2); + assert_eq!(host.state.get(), 0); + assert_eq!(host.recovery.generation(), 1); + assert!(!host.blamed); + } + + #[test] + fn unknown_25g83_keys_still_preserve_firmware_blame() { + let mut host = Halted::new(false); + service(&mut host, || false).unwrap(); + assert_eq!(host.reads.get(), 0); + assert!(host.blamed); + assert_eq!(host.state.get(), 0); + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index c30e45e22579a..d71d3cc81553f 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -2,17 +2,52 @@ //! Fail closed on repeated watchdog/restart traffic, even across client loss. +use core::sync::atomic::{AtomicU64, Ordering}; + const WINDOW_NS: u64 = 30_000_000_000; const LIMIT: u8 = 8; +/// Commands retired successfully. A watchdog storm means the GPU stopped making progress; +/// timeouts of overloaded clients while other work keeps retiring are not one. +static RETIRED: AtomicU64 = AtomicU64::new(0); +/// [`crate::g17::now_ns`] of the latest successful retirement, zero before the first. +static LAST_RETIRED_NS: AtomicU64 = AtomicU64::new(0); + +/// Called for every command that retires without error. +pub(crate) fn note_retired() { + RETIRED.fetch_add(1, Ordering::Relaxed); + LAST_RETIRED_NS.store(crate::g17::now_ns(), Ordering::Relaxed); +} + +pub(super) fn retired() -> u64 { + RETIRED.load(Ordering::Relaxed) +} + +/// Nanoseconds since any command last retired successfully, or `None` before the first. +pub(crate) fn since_retired(now: u64) -> Option { + match LAST_RETIRED_NS.load(Ordering::Relaxed) { + 0 => None, + last => Some(now.saturating_sub(last)), + } +} + pub(super) struct Storm { start: Option, count: u8, stopped: bool, + progress: u64, } impl Storm { pub(super) const fn new() -> Self { - Self { start: None, count: 0, stopped: false } + Self { start: None, count: 0, stopped: false, progress: 0 } + } + /// Like [`Self::note`], but work retired since the previous event starts a new window. + pub(super) fn note_unless_progress(&mut self, now: u64, progress: u64) -> bool { + if !self.stopped && progress != self.progress { + self.progress = progress; + self.start = None; + } + self.note(now) } pub(super) fn note(&mut self, now: u64) -> bool { if self.stopped { return true; } @@ -74,11 +109,23 @@ mod tests { for _ in 0..1024 { assert!(storm.note(u64::MAX)); } } #[test] + fn recovery_table_ends_at_its_first_invalid_record() { + use crate::g17::recovery::Entries; + let mut entries = Entries::default(); + assert!(entries.include(1 | (12 << 4))); + assert!(entries.include(1 | 2 | (17 << 4))); + assert!(!entries.include(0)); + assert_eq!(entries.listed, 2); + assert_eq!(entries.guilty, 1u128 << 17); + assert_eq!(entries.cancelled, 0); + } + #[test] fn replay_dependency_checks_never_wait_for_event_service() { use crate::g17::dependency::replay_status; assert_eq!(replay_status(0), Err(EBUSY)); assert_eq!(replay_status(1), Ok(())); assert_eq!(replay_status(EIO.to_errno()), Err(ECANCELED)); assert_eq!(replay_status(ETIMEDOUT.to_errno()), Err(ECANCELED)); + assert_eq!(replay_status(ENODATA.to_errno()), Err(ECANCELED)); } } diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs index 94daa5ec0dfeb..7668871376bad 100644 --- a/drivers/gpu/drm/asahi/g17/replay.rs +++ b/drivers/gpu/drm/asahi/g17/replay.rs @@ -16,6 +16,12 @@ pub(crate) struct Group { pub(crate) abandoned: bool, /// Already republished the maximum number of times. pub(crate) exhausted: bool, + /// Retired by firmware after executing in full. + pub(crate) ran: bool, + /// Retired by firmware unexecuted, but never to run again: a recovery interrupted it after + /// its tiler started, so it may have run partly, or a command of another queue waited on it, + /// and that wait ended when firmware retired the group, before its output existed. + pub(crate) lost: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -25,26 +31,54 @@ pub(crate) enum Decision { Wait, /// Never replay this set; the render timeout settles it. Refuse, + /// Firmware has retired every group, but a dropped group cannot run again in order: a later + /// group already ran, or the dropped group had started or had a consumer. Fail the set now. + Fail, } /// Groups are read in order and the first one that decides stops the walk, so a caller can /// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its /// successors, which consume its output, with it. An exhausted successor would break the order. +/// +/// After a restart the firmware retires the groups it had started on a pair without running +/// them, while groups it had not yet started run as usual. Once such a later group has run, a +/// replay of the dropped ones would run them after it; a group that had started cannot run +/// twice; and a group another queue waited on may already have been read before it ran. None of +/// these is replayed: the whole set fails at once instead of waiting for the render timeout. A +/// group that ran ahead of every dropped one is left to in-order retirement. pub(crate) fn decide(groups: impl Iterator) -> Decision { let mut front_exhausted = None; let mut successor_exhausted = false; + let mut dropped = false; + let mut unordered = false; for group in groups { if !group.innocent || group.blamed { return Decision::Refuse; } + if group.ran { + if !dropped { + return Decision::Wait; + } + unordered = true; + continue; + } + if group.lost { + dropped = true; + unordered = true; + continue; + } if !group.abandoned { return Decision::Wait; } + dropped = true; match front_exhausted { None => front_exhausted = Some(group.exhausted), Some(_) => successor_exhausted |= group.exhausted, } } + if unordered { + return Decision::Fail; + } match front_exhausted { Some(true) => Decision::Replay, Some(false) if !successor_exhausted => Decision::Replay, @@ -56,10 +90,13 @@ pub(crate) fn decide(groups: impl Iterator) -> Decision { mod tests { use super::*; fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { - Group { innocent, blamed: false, abandoned, exhausted } + Group { innocent, blamed: false, abandoned, exhausted, ran: false, lost: false } } - const OK: Group = Group { innocent: true, blamed: false, abandoned: true, exhausted: false }; - const BLAMED: Group = Group { innocent: true, blamed: true, abandoned: true, exhausted: false }; + const OK: Group = + Group { innocent: true, blamed: false, abandoned: true, exhausted: false, ran: false, lost: false }; + const BLAMED: Group = Group { blamed: true, ..OK }; + const RAN: Group = Group { abandoned: false, ran: true, ..OK }; + const LOST: Group = Group { abandoned: false, lost: true, ..OK }; #[test] fn every_group_abandoned_and_innocent_is_replayed() { assert_eq!(decide([OK].into_iter()), Decision::Replay); @@ -106,4 +143,28 @@ mod tests { Decision::Replay ); } + #[test] + fn a_group_that_ran_after_dropped_ones_fails_the_set() { + // Seen after restarts: the started groups were dropped, the queued ones ran. + assert_eq!(decide([OK, OK, RAN, RAN].into_iter()), Decision::Fail); + assert_eq!(decide([OK, RAN, OK].into_iter()), Decision::Fail); + // Also with an exhausted front, which a replay would only fail later. + assert_eq!(decide([group(true, true, true), RAN].into_iter()), Decision::Fail); + } + #[test] + fn a_group_that_ran_first_is_left_to_retirement() { + assert_eq!(decide([RAN].into_iter()), Decision::Wait); + assert_eq!(decide([RAN, OK].into_iter()), Decision::Wait); + } + #[test] + fn a_started_dropped_group_fails_the_set() { + assert_eq!(decide([OK, LOST].into_iter()), Decision::Fail); + assert_eq!(decide([LOST, OK].into_iter()), Decision::Fail); + } + #[test] + fn failing_still_waits_for_live_groups_and_refuses_guilty_ones() { + assert_eq!(decide([OK, RAN, group(true, false, false)].into_iter()), Decision::Wait); + assert_eq!(decide([OK, RAN, group(false, true, false)].into_iter()), Decision::Refuse); + assert_eq!(decide([LOST, BLAMED].into_iter()), Decision::Refuse); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6f4a5355d6eb3..d391fbb3001bb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -31,9 +31,32 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; -const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; +/// Pending admission keeps exactly retired compute backing available for reuse. +/// The guard spans retries and disappears on publication, cancellation or error. +/// It borrows Shared, so the accounting cannot retain the device after teardown. +struct ComputeDemand<'a>(&'a Arc); + +impl<'a> ComputeDemand<'a> { + fn new(shared: &'a Arc) -> Self { + // Each admitted logical queue has at most one scheduler publisher. + // The execution-context bound is far below the counter's range. + shared.compute_demand.fetch_add(1, Ordering::AcqRel); + Self(shared) + } +} + +impl Drop for ComputeDemand<'_> { + fn drop(&mut self) { + if self.0.compute_demand.fetch_sub(1, Ordering::AcqRel) == 1 { + // Reinstate normal idle retention even if the last publisher was + // cancelled before a kick could produce another event. + self.0.queue_events(); + } + } +} + /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. #[derive(Clone, Copy)] @@ -45,13 +68,103 @@ pub(super) struct ComputeReservation { struct ComputeEntry { reservation: ComputeReservation, - queue: Option>, + queue: ComputeSlot, +} + +/// A physical compute queue behind its own lock. Publication into a bound queue takes only +/// this lock; device-mutex holders take it to observe or change the queue (device mutex -> +/// cell). A logical queue may keep the cell after the registry gave the queue up: the cell is +/// then empty, so the cached reference never extends any firmware-visible lifetime. +#[pin_data] +pub(super) struct ComputeCell { + #[pin] + queue: kernel::sync::Mutex>>, +} + +impl ComputeCell { + fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + queue <- kernel::new_mutex!(None, "g17::ComputeCell::queue"), + }), + GFP_KERNEL, + ) + } + + /// The installed queue, locked; None once the registry gave it up. + pub(super) fn lock(&self) -> Option> { + QueueGuard::new(self.queue.lock()) + } +} + +/// A locked installed queue. +pub(super) struct QueueGuard<'a> { + queue: core::ptr::NonNull, + _guard: kernel::sync::MutexGuard<'a, Option>>, +} + +impl<'a> QueueGuard<'a> { + fn new(mut guard: kernel::sync::MutexGuard<'a, Option>>) -> Option { + let queue = core::ptr::NonNull::from((*guard).as_deref_mut()?); + Some(Self { queue, _guard: guard }) + } +} + +impl core::ops::Deref for QueueGuard<'_> { + type Target = compute::Queue; + fn deref(&self) -> &compute::Queue { + // SAFETY: The boxed queue cannot be taken or dropped while the guard holds its lock. + unsafe { self.queue.as_ref() } + } +} + +impl core::ops::DerefMut for QueueGuard<'_> { + fn deref_mut(&mut self) -> &mut compute::Queue { + // SAFETY: As for `deref`; the lock makes this the only reference. + unsafe { self.queue.as_mut() } + } +} + +/// The registry's handle on one compute cell. Dropping it drops the queue in the cell, so a +/// cached cell never keeps a removed queue (or its QID's aliases) alive. +struct ComputeSlot(Arc); + +impl ComputeSlot { + fn new() -> Result { + Ok(Self(ComputeCell::new()?)) + } + fn as_deref(&self) -> Option> { + self.0.lock() + } + fn as_deref_mut(&mut self) -> Option> { + self.0.lock() + } + fn is_some(&self) -> bool { + self.0.queue.lock().is_some() + } + fn is_none(&self) -> bool { + !self.is_some() + } + /// Gives the queue up to the caller, which drops it outside the cell lock. + fn take(&mut self) -> Option> { + self.0.queue.lock().take() + } + fn put(&mut self, queue: Option>) { + let old = core::mem::replace(&mut *self.0.queue.lock(), queue); + drop(old); + } +} + +impl Drop for ComputeSlot { + fn drop(&mut self) { + let queue = self.0.queue.lock().take(); + drop(queue); + } } pub(super) struct Registry { pub(super) qids: kick::QueueIds, pub(super) buffers: BufferIds, - pub(super) accounting: qos::Accounting, pub(super) metrics: Arc, pub(super) qos_ids: Arc, pub(super) admission: Arc, @@ -62,6 +175,10 @@ pub(super) struct Registry { compute_scheduler_building: bool, replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, + /// Completion-ring slot where the next compute scan starts. + completion_cursor: usize, + /// Control word of each completion-ring slot as of the last scan that read it. + completion_seen: KVVec, worker_deferred: Option, teardown: super::teardown::Pending, } @@ -69,21 +186,34 @@ pub(super) struct Registry { impl Registry { /// Initialize inside the firmware allocation instead of staging the whole /// registry on the probe stack. No additional allocation is required. + #[cfg(CONFIG_KUNIT = "y")] pub(super) fn new( clusters: u32, deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + ) -> impl Init { + Self::with_waiters(clusters, deflake, descriptor_flags, variant, None, None) + } + + /// `queues` wakes logical-queue creation when a logical queue slot is returned; + /// `renders` wakes render publication when a render pair or lease returns. + pub(super) fn with_waiters( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, + queues: Option>, + renders: Option>, ) -> impl Init { kernel::try_init!(Self { qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { qids.reserve_initial_render() }), buffers <- BufferIds::new(), - accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, + admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { @@ -97,12 +227,21 @@ impl Registry { compute_scheduler_building: false, replay_outer_wait: None, observations: compute::retirement::Batch::new()?, + completion_cursor: 0, + completion_seen: { + let slots = super::initdata::COMPUTE_COMPLETION_SLOTS; + let mut seen = KVVec::with_capacity(slots, GFP_KERNEL)?; + for _ in 0..slots { + seen.push_within_capacity(0).map_err(|_| ENOMEM)?; + } + seen + }, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) } - pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { + pub(super) fn compute(&self, owner: u64) -> Option> { self.compute .iter() .flatten() @@ -110,12 +249,16 @@ impl Registry { .find(|queue| queue.owner() == Some(owner)) } - pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { - self.compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.owner() == Some(owner)) + pub(super) fn compute_mut(&mut self, owner: u64) -> Option> { + self.compute(owner) + } + + /// The cell of `owner`'s bound queue, which its publisher may lock without the device mutex. + pub(super) fn compute_cell(&self, owner: u64) -> Option> { + self.compute.iter().flatten().find_map(|entry| { + entry.queue.as_deref()?.owner().filter(|bound| *bound == owner)?; + Some(entry.queue.0.clone()) + }) } /// An installed idle queue keeps its original QID and backing. Mapping a @@ -131,15 +274,80 @@ impl Registry { Rebind::Fresh }); } - // Physical channels are a device budget, not a permanent entitlement - // of each open logical queue. Only exact idle retirement can yield one. - for queue in self.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { - admission.check()?; - queue.share_idle()?; + // A binding persists for its logical queue's lifetime while physical + // QIDs allow: take a channel of a closed owner, then a fresh QID, and + // only then the least recently idle channel of one open owner. + if let Some(rebind) = self.bind_reusable(owner, context, required, admission)? { + return Ok(rebind); } + if self.qids.compute_fits(self.admission.queues() as usize) { + return Ok(Rebind::Fresh); + } + let teardown = &self.teardown; + let idle = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.shareable()) + .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) + .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) + .min_by_key(|(key, _)| *key) + .map(|(_, qid)| qid); + let Some(qid) = idle else { return Err(ENOSPC); }; + admission.check()?; + let shared = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)? + .share_idle()?; + if !shared { + return Err(ENOSPC); + } + Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) + } + + /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks + /// only whether some channel is free for a new owner. + fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { + if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { + return if queue.spared_pending() { + Readiness::Progress + } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { + // Publication reports the mismatch. + Readiness::Ready + } else if queue.pool_fits(required) || !queue.growing() { + Readiness::Ready + } else { + Readiness::Progress + }; + } + let teardown = &self.teardown; + let available = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| { + (queue.reusable() || queue.shareable() || queue.can_rearm()) + && !queue.context().is_some_and(|bound| teardown.releasing(bound)) + }); + if available || self.qids.compute_fits(self.admission.queues() as usize) { + Readiness::Ready + } else { + Readiness::Channel + } + } + + /// Binds `owner` to a reusable channel whose last owner released it, or + /// names a vacant one to re-arm. None if no channel is reusable. + fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { let mut vacant = None; let teardown = &self.teardown; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -159,9 +367,9 @@ impl Registry { } if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; - return Ok(Rebind::Done); + return Ok(Some(Rebind::Done)); } - Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) + Ok(vacant.map(Rebind::Rearm)) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -172,7 +380,7 @@ impl Registry { pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -184,7 +392,7 @@ impl Registry { pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { let teardown = &self.teardown; - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -252,7 +460,7 @@ impl Registry { let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -282,7 +490,7 @@ impl Registry { let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); let mut freed = 0; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -313,7 +521,7 @@ impl Registry { || self.compute.iter().flatten().any(|entry| { entry .queue - .as_ref() + .as_deref() .map_or(entry.reservation.owner == owner, |queue| { queue.owner() == Some(owner) }) @@ -328,14 +536,16 @@ impl Registry { .ok_or_else(|| self.compute_admission_error( crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; let identity = self.identities.reserve(1)?.start; + let queues = self.admission.queues() as usize; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .reserve_for(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID), queues) .map_err(|error| self.compute_admission_error(error))?; + self.note_build_room(); let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, - queue: None, + queue: ComputeSlot::new()?, }); Ok(reservation) } @@ -367,7 +577,7 @@ impl Registry { return Err(EIO); } let entry = self.reserved(reservation)?; - entry.queue = queue.take(); + entry.queue.put(queue.take()); Ok(()) } @@ -375,6 +585,7 @@ impl Registry { self.reserved(reservation)?; self.qids.cancel_unpublished(reservation.id)?; self.compute[reservation.slot] = None; + self.note_build_room(); Ok(()) } @@ -382,20 +593,32 @@ impl Registry { let Some(slot) = self.compute.iter().position(|entry| { entry .as_ref() - .and_then(|entry| entry.queue.as_ref()) + .and_then(|entry| entry.queue.as_deref()) .is_some_and(|queue| queue.owner() == Some(owner)) }) else { return Ok(()); }; let entry = self.compute[slot].as_mut().ok_or(EIO)?; - if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { + let unpublished = entry.queue.as_deref_mut().ok_or(EIO)?.release_owner()?; + if unpublished { self.qids.cancel_unpublished(entry.reservation.id)?; self.compute[slot] = None; + self.note_build_room(); } Ok(()) } } +/// What a compute packet still lacks before publication. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(super) enum Readiness { + Ready, + /// A physical channel: every one is busy and no QID fits. + Channel, + /// Retirement-bound capacity or reopened publication. + Progress, +} + /// Outcome of offering an owner a retained compute queue. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Rebind { @@ -436,12 +659,28 @@ mod uninstalled_cleanup_tests { /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. +#[pin_data] pub(super) struct Backend { shared: Arc, pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + compute_backlog: core::sync::atomic::AtomicU32, closed: core::sync::atomic::AtomicBool, + /// The cell of the compute queue this owner last published to under the device mutex, + /// with the shared compute objects mapped into its VM. Direct publication revalidates the + /// binding under the cell lock; a stale cell only sends the packet back to that path. + #[pin] + direct: kernel::sync::Mutex>>, + /// The channel a prepared packet is about to publish into. Its claim keeps the idle + /// channel from being shared away between prepare_job and run_job. + #[pin] + claimed: kernel::sync::Mutex>>, + /// Render pair admission, shared by every logical queue of the session. + admission: Arc, + /// The render pair this owner last published to without the device mutex + /// (zero for none): a hint, revalidated by every lend. + direct_slot: core::sync::atomic::AtomicU32, } impl Backend { @@ -459,6 +698,7 @@ impl Backend { const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; let started = now_ns(); let (lease, uat, pool, qos, execution, slot) = loop { + let observed = shared.queues.observe(); let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -466,6 +706,11 @@ impl Backend { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; + // Installed compute channels narrow render's QIDs: every open + // logical queue keeps a dedicated render pair within reach. + if !firmware.queues.qids.render_fits(firmware.queues.admission.queues() as usize) { + return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); + } match firmware.uat.new_execution_context(vm) { Ok(execution) => { break ( @@ -493,7 +738,10 @@ impl Backend { return Err(ERESTARTSYS); } shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + // A closing queue returns its context when its release + // is reclaimed; that reclaim wakes this creator. + let left = RETAINED_CONTEXT_WAIT_NS.saturating_sub(now_ns().saturating_sub(started)); + shared.queues.block(observed, left.div_ceil(1_000_000) as u32, true)?; } Err(error) => return Err(error), } @@ -506,7 +754,16 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, + Context::new( + &alloc, + &pool, + &qos, + execution, + owner_pid, + policy, + shared.renders.clone(), + shared.progress.clone(), + )?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -514,15 +771,21 @@ impl Backend { return Err(EAGAIN); } drop(state); + let admission = slot.pool().clone(); Ok(( - Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { shared: shared.clone(), context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + compute_backlog: core::sync::atomic::AtomicU32::new(0), closed: core::sync::atomic::AtomicBool::new(false), - }, + direct <- kernel::new_mutex!(None, "g17::Backend::direct"), + claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), + admission, + direct_slot: core::sync::atomic::AtomicU32::new(0), + }), GFP_KERNEL, )?, slot, @@ -592,7 +855,7 @@ impl Backend { .queues .compute_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); drop(lease); if result != Err(EAGAIN) { return result; @@ -652,7 +915,7 @@ impl Backend { .queues .compute_scheduler_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); result } @@ -667,11 +930,14 @@ impl Backend { drop(queue); let mut state = self.shared.state.lock(); let firmware = state.as_deref_mut().ok_or(ENODEV)?; - firmware.queues.cancel_compute(reservation) + firmware.queues.cancel_compute(reservation)?; + drop(state); + // The slot and its never-published QID are free again. + self.shared.channels.notify(); + Ok(()) } fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { - use kernel::time::{delay::fsleep, Delta}; admission.check()?; { let state = self.shared.state.lock(); @@ -689,6 +955,9 @@ impl Backend { let mut deferred = DeferredBatch::publication()?; let mut growth = None; let result = (|| loop { + // Growth waits for this queue's retirement and for firmware to consume the pool + // release; both are progress the event worker reports. + let observed = self.shared.progress.observe(); let result = (|| { let mut state = self.shared.state.lock(); admission.check()?; @@ -696,7 +965,7 @@ impl Backend { if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; + let mut queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; if queue.pool_fits(required) { return Ok(None); } if growth.is_none() { let notify = queue.begin_growth(&firmware.init, required)?; @@ -714,9 +983,9 @@ impl Backend { match result { Ok(old) => { drop(old); return Ok(()); } Err(error) if error == EAGAIN && lease.is_current() => { - self.shared.queue_events(); - fsleep(Delta::from_millis(1)); - } + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; + } Err(error) => return Err(error), } })(); @@ -727,8 +996,9 @@ impl Backend { if let Some(firmware) = state.as_deref_mut() { for index in 0..firmware.queues.compute.len() { let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; - let Some(queue) = entry.queue.as_deref_mut() else { continue; }; - if queue.abort_growth(token)? == Some(true) { + let Some(aborted) = entry.queue.as_deref_mut() + .map(|mut queue| queue.abort_growth(token)).transpose()? else { continue; }; + if aborted == Some(true) { // Keep the empty entry and QID reserved until all // graph aliases have been destroyed outside the lock. retired = entry.queue.take().map(|queue| (entry.reservation, queue)); @@ -762,9 +1032,11 @@ impl Backend { } loop { admission.check()?; + // Sampled before any capacity check of this attempt. + let observed = self.shared.channels.observe(); let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; - let (uat, reservation) = { + let (uat, objects, reservation) = { let mut state = self.shared.state.lock(); admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -780,7 +1052,7 @@ impl Backend { Err(error) if error == ENOSPC => { drop(state); drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, true)?; continue; } result => result?, @@ -788,16 +1060,17 @@ impl Backend { match rebound { Rebind::Done => return Ok(()), Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } - Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), + Rebind::Rearm(qid) => (firmware.uat.clone(), firmware.pool.clone(), Err(qid)), Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { Ok(reservation) => reservation, Err(error) if error == EAGAIN => { - // The event worker owns release observation and - // drops vacated aliases/backing off this mutex. + // All physical channels are occupied. Retain their + // USC backing for exact-retirement handoff; releasing + // it cannot create a QID and forces fresh allocation. drop(state); drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, false)?; continue; } Err(error) if error == ENOSPC => { @@ -811,7 +1084,7 @@ impl Backend { } Err(error) => return Err(error), }; - (firmware.uat.clone(), Ok(reservation)) + (firmware.uat.clone(), firmware.pool.clone(), Ok(reservation)) } } }; @@ -826,12 +1099,12 @@ impl Backend { // queue may have been taken or re-armed by another owner since. let built = (|| -> Result<_> { let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - Ok((pages, compute::DescriptorRing::new(&alloc)?)) + Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) })(); let (pages, ring) = match built { Err(error) if error == ENOSPC => { drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, true)?; continue; } result => result?, @@ -881,6 +1154,7 @@ impl Backend { self.compute_scheduler(&alloc, &lease, admission)?; let queue = compute::Queue::new( &alloc, + &objects, backing, self.owner, self.context.clone(), @@ -920,7 +1194,7 @@ impl Backend { } drop(lease); if result == Err(ENOSPC) { - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, true)?; continue; } if result != Err(EAGAIN) { return result; } @@ -932,6 +1206,7 @@ impl Backend { /// The physical namespace bounds the batch independently of open logical queues. /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { + has_source_completions: bool, values: KVVec, /// Exited owners detached from idle retained compute queues and the backing /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. @@ -967,6 +1242,7 @@ impl DeferredBatch { 0 }; Ok(Self { + has_source_completions: false, values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, @@ -1020,10 +1296,18 @@ impl DeferredBatch { // it. Expose terminal VM status now, before off-lock fence callbacks. // Recovery-spared and cancelled commands never poison their VM. if let Deferred::FailedOwned(completion, error) = &value { - if *error != ECANCELED && !completion.spared() { + if completion.poisons(*error) { completion.status().record(*error); } } + match &value { + Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) + | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => + { completion.note_input_error(*error); }, + Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => + completion.note_source_retired(), + _ => {}, + } self.values.push_within_capacity(value).map_err(|_| EIO)?; // A later publisher can borrow this pair before finish signals its old // fence. Publish the exact completion's witness only after collection @@ -1035,11 +1319,33 @@ impl DeferredBatch { ) = self.values.last() { completion.note_retirement_queued(); + self.has_source_completions |= completion.sources().is_some(); } Ok(()) } + /// Completions collected so far. + pub(super) fn pending(&self) -> usize { + self.values.len() + } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { + use super::completion::Deferred; + if self.has_source_completions { + for _ in 0..self.values.len() { + let mut changed = false; + for value in &self.values { + if let Deferred::Retired(completion, Ok(_)) + | Deferred::Drained(completion, Ok(_)) = value + { + if let Err(error) = completion.check_source_inputs() { + changed |= completion.note_input_error(error); + } + } + } + if !changed { break; } + } + } + self.has_source_completions = false; for value in self.values.drain_all() { value.finish(); } @@ -1060,15 +1366,15 @@ impl DeferredBatch { struct ComputeHost<'a> { init: &'a super::initdata::InitData, - primary: &'a mut super::Coprocessor, + port: &'a super::port::Port, + outer: Option>, effort: &'a mut super::power::Effort, qids: &'a mut kick::QueueIds, - accounting: &'a mut qos::Accounting, generation: u64, } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - self.qids.next_publication() + Ok(self.port.next_publication()) } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -1085,54 +1391,151 @@ impl compute::Host for ComputeHost<'_> { scheduler: u64, policy: super::fw::queue::Policy, ) -> Result { - self.accounting.publish( - &qos::View::new(self.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + self.port.qos_publish(owner, scheduler, policy) } fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.accounting - .cancel(&qos::View::new(self.init.qos()?)?, publication) + self.port.qos_cancel(publication) } fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.init.publish_work(priority, slot).map(|_| ()) + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { - self.primary.notify(message) + self.port.notify(message) } fn note_submission(&mut self) -> Result { self.effort.submit(); - super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) + super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) + } +} + +/// Publication into a bound, installed queue through the port alone. The caller checked the +/// recovery state, full effort and the queue's binding; first installation never comes here. +struct DirectHost<'a> { + port: &'a super::port::Port, + outer: Option>, + generation: u64, +} +impl compute::Host for DirectHost<'_> { + fn next_compute_publication(&mut self) -> Result { + Ok(self.port.next_publication()) + } + fn epoch(&self) -> Result<(u64, u32)> { + Ok((self.generation, self.port.recovery_state()?)) + } + fn prepare_compute_shared(&mut self) -> Result { + Err(EAGAIN) + } + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: super::fw::queue::Policy, + ) -> Result { + self.port.qos_publish(owner, scheduler, policy) + } + fn qos_cancel(&mut self, publication: qos::Publication) -> Result { + self.port.qos_cancel(publication) + } + fn publish_qid(&mut self, _id: kick::Id) -> Result { + // An installed queue published its identity with its first command. + Ok(()) + } + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) + } + fn notify(&mut self, message: u64) -> Result { + self.port.notify(message) + } + fn note_submission(&mut self) -> Result { + Ok(()) } } impl super::Firmware { - fn publish_compute( + #[inline] + fn publish_compute(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if packet.completion.sources().is_some() { + self.publish_compute_inputs(owner, packet, dependencies, deferred) + } else { + self.publish_compute_raw(owner, packet, dependencies, &[], deferred) + } + } + + #[inline(never)] + fn publish_compute_inputs(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if self.recovery.pending() || super::recovery::Memory::recovery_state(&self.init)? != 0 { + return Err(EAGAIN); + } + let sources = packet.completion.sources().ok_or(EINVAL)?.clone(); + sources.check()?; + let mut merged = super::dependency::Frontier::new(); + for dependency in dependencies { merged.insert(0, *dependency)?; } + let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + let mut native: KVec = + KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + for source in &sources.sources { + if source.submission.source_status()? { continue; } + let found = self.queues.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find_map(|mut queue| match queue.source_dependency(source, packet.completion.status(), + self.recovery.generation()) { + Ok(None) => None, + result => Some(result), + }).ok_or(EAGAIN)??; + let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { + return Err(EAGAIN); + }; + // The completion-wait record orders the consumer behind its + // source; naming the source in the kick entry as well only adds + // a second, slower wait and a predecessor record. + let wait = super::sync_input::NativeWait { sequence, dependency, stamp_va, event_slot }; + if let Some(old) = native.iter_mut().find(|old| + old.dependency.qid() == dependency.qid()) { + if sequence > old.sequence { *old = wait; } + } else { + native.push(wait, GFP_KERNEL)?; + } + holds.push(hold, GFP_KERNEL)?; + } + let result = self.publish_compute_raw(owner, packet.clone(), merged.as_slice(), + native.as_slice(), deferred); + if packet.is_published() { + for queue in self.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) + { + for source in &sources.sources { queue.note_input_consumer(source); } + } + sources.set_holds(holds); + super::sync_input::note_published(); + super::sync_input::note_native(native.len()); + } + result + } + + fn publish_compute_raw( &mut self, owner: u64, packet: Arc, dependencies: &[super::fw::kick::KickDependency], + native: &[super::sync_input::NativeWait], deferred: &mut DeferredBatch, ) -> Result { if self.recovery.pending() { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // capacity check. The same device lock covers queue.publish below. - if !self - .init - .work_ready(packet.context.policy().priority(), DataMaster::Compute)? - { - return Err(EAGAIN); - } + // reservation; dropping it unused returns the work-ring slot. + let outer = self.port.reserve_compute(packet.context.policy().priority())?; let queues = &mut self.queues; - let queue = queues + let mut queue = queues .compute .iter_mut() .flatten() @@ -1150,26 +1553,42 @@ impl super::Firmware { } let mut host = ComputeHost { init: &self.init, - primary: &mut self.primary, + port: &self.port, + outer: Some(outer), effort: &mut self.effort, qids: &mut queues.qids, - accounting: &mut queues.accounting, generation: self.recovery.generation(), }; - queue.publish(packet, dependencies, &mut host, &mut |event| { - deferred.push(event) - }) + if native.is_empty() { + queue.publish(packet, dependencies, &mut host, &mut |event| deferred.push(event)) + } else { + queue.publish_with_native_waits(packet, dependencies, native, + &mut host, &mut |event| deferred.push(event)) + } } } impl Registry { + /// Everything ensure_compute establishes for `owner`: the shared compute + /// objects mapped into its VM, and its queue bound to `context` with a + /// pool that fits and no spared witness to release. + fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { + let Some(queue) = self.compute(owner) else { + return Ok(false); + }; + Ok(!queue.spared_pending() + && queue.pool_fits(required) + && queue.matches(owner, queue.qid(), context)?) + } + fn release_compute_witness( &mut self, init: &super::initdata::InitData, + port: &super::port::Port, owner: u64, deferred: &mut DeferredBatch, ) -> Result { - let Some(queue) = self + let Some(mut queue) = self .compute .iter_mut() .flatten() @@ -1181,10 +1600,9 @@ impl Registry { if !queue.spared_pending() { return Ok(false); } - let view = qos::View::new(init.qos()?)?; queue.release_witnessed( super::recovery::Memory::recovery_state(init)?, - |owner| self.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; Ok(queue.spared_pending()) @@ -1196,7 +1614,7 @@ impl Registry { error: Error, deferred: &mut DeferredBatch, ) -> Result { - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -1223,33 +1641,53 @@ impl super::Firmware { let registry = &mut self.queues; registry.observations.clear(); let mut requested = false; + // Every queue's pending identities are staged, not only the ones this batch + // retires: the scan passes each new record once, and a queue retired by a later + // batch finds its observation already latched. for queue in registry .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) { - requested |= queue.request_observations(&mut registry.observations, masks)?; + requested |= queue.request_observations(&mut registry.observations)?; } + // Completion records are appended in order: fold only the records written since + // the previous scan. A request whose stamp already covers it has its record in the + // ring; if the scan ended before finding one, read the whole ring once. if requested { - self.init - .scan_compute_completions(|record| registry.observations.fold(record))?; + let observations = &mut registry.observations; + let seen = &mut registry.completion_seen; + registry.completion_cursor = self.init.fold_compute_completions( + registry.completion_cursor, + seen, + |record| observations.fold(record), + )?; + if observations.covered_pending() { + registry.completion_cursor = self + .init + .resync_compute_completions(seen, |record| observations.fold_any(record))?; + } } - let view = qos::View::new(self.init.qos()?)?; - for queue in registry + let port = &self.port; + let retired = deferred.pending(); + let mut freed = false; + for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.latch_observations(&mut registry.observations, masks)?; + let busy = !queue.channel_free(); + queue.latch_observations(&mut registry.observations)?; queue.retire( masks, generation, state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. @@ -1257,26 +1695,37 @@ impl super::Firmware { let terminal = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .and_then(compute::Queue::take_terminal); + .and_then(|mut queue| queue.take_terminal()); if let Some((status, error)) = terminal { status.record(error); self.settle_render_vm(&status, error, deferred)?; self.queues.fail_compute_vm(&status, error, deferred)?; } } - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.release_witnessed( state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); + } + // Wake only the admission waits this batch can end. The fences gate + // scheduler entities alone; their callbacks never take the device mutex. + let shared = &self.primary.state.shared; + if freed { + shared.channels.notify(); + } + if deferred.pending() != retired { + shared.progress.notify(); } Ok(()) } @@ -1286,18 +1735,17 @@ impl Backend { fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { admission.check()?; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; - use kernel::time::{delay::fsleep, Delta}; - const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; - for _ in 0..SPARED_WAIT_MS { + loop { admission.check()?; + let observed = self.shared.progress.observe(); let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.replay_compute(&mut deferred)?; firmware .queues - .release_compute_witness(&firmware.init, self.owner, &mut deferred) + .release_compute_witness(&firmware.init, &firmware.port, self.owner, &mut deferred) })(); deferred.finish(); let pending = pending?; @@ -1307,14 +1755,308 @@ impl Backend { result => return result, } } - // Other owners continue submitting while this owner gives its - // queue-local drain witness time to arrive. - fsleep(Delta::from_millis(1)); + // Other owners continue submitting while this owner waits for its + // queue-local drain witness. No new command has been accepted; if the + // admission deadline passes, the spared queue and its retained work stay + // intact until a later admission observes its witness. + admission.extend_for_recovery(); + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; } - // No new command has been accepted. Keep the spared queue and its - // retained work intact until a later admission observes its witness. - admission.check()?; - Err(EBUSY) + } + + /// Steady-state publication into this owner's bound channel without the device mutex: only + /// the channel's own lock and the port's small locks are taken, and nothing device-wide is + /// scanned. None leaves the packet untouched for the device-mutex path, which owns binding, + /// first installation, input sources, render prefixes, replay and recovery. + fn compute_direct( + &self, + packet: &Arc, + required: usize, + deferred: &mut DeferredBatch, + ) -> Option { + if packet.completion.sources().is_some() + || self.context.render_seen() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let cell = self.direct.lock().clone()?; + let lease = self.shared.preparations.try_enter().ok()?; + let port = self.shared.port(&lease)?; + if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { + return None; + } + // Full effort is applied by the device-mutex path when it is not already. + if !port.note_submission() { + return None; + } + match packet.completion.work_state().ok()?.acquire(crate::g17::job::Engine::Compute) { + Ok(_) => {} + Err(error) if error == EBUSY => return None, + Err(error) => return Some(Err(error)), + } + let mut queue = cell.lock()?; + if !queue.installed() + || queue.spared_pending() + || !queue.pool_fits(required) + || queue.pending_switch_from().is_some() + || !matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)) + || !lease.is_current() + { + return None; + } + let outer = port.reserve_compute(packet.context.policy().priority()).ok()?; + let mut host = DirectHost { + port, + outer: Some(outer), + generation: port.generation(), + }; + match queue.publish(packet.clone(), &[], &mut host, &mut |event| deferred.push(event)) { + Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, + result => Some(result), + } + } + + /// Scheduler admission of a compute packet. Shared capacity the packet lacks (a channel, a + /// work state, retirement-bound room, reopened publication) becomes a fence that the release + /// providing it signals, so publication finds it ready instead of sleeping for it. Errors + /// and anything prepare cannot judge are left to publication. + pub(super) fn prepare_compute(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()).ok()?; + loop { + let channels = self.shared.channels.observe(); + let progress = self.shared.progress.observe(); + let (domain, observed) = match self.compute_readiness(packet, required) { + Readiness::Ready => return None, + Readiness::Channel => (&self.shared.channels, channels), + Readiness::Progress => (&self.shared.progress, progress), + }; + self.shared.queue_events(); + match domain.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + + fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { + match packet.completion.work_state().and_then(|work| work.acquire(crate::g17::job::Engine::Compute)) { + Err(error) if error == EBUSY => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(_) => {} + } + // The steady state: this owner's channel is bound and installed. + if packet.completion.sources().is_none() { + if let Some(cell) = self.direct.lock().clone() { + let bound = cell.lock().is_some_and(|mut queue| { + let bound = queue.installed() + && !queue.spared_pending() + && queue.pool_fits(required) + && matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)); + if bound { + queue.claim(); + } + bound + }); + if bound { + self.hold_claim(Some(cell)); + return Readiness::Ready; + } + } + } + match self.sources_published(packet) { + Ok(false) => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(true) => {} + } + let lease = match self.shared.preparations.try_enter() { + Ok(lease) => lease, + Err(error) if error == EAGAIN => return Readiness::Progress, + Err(_) => return Readiness::Ready, + }; + let state = self.shared.state.lock(); + let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; + if !lease.is_current() + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 + { + return Readiness::Progress; + } + firmware.queues.compute_readiness(self.owner, Some(&self.context), required) + } + + /// Replaces the prepared packet's channel claim; None releases it after publication, + /// cancellation or close. + pub(super) fn hold_claim(&self, cell: Option>) { + let old = core::mem::replace(&mut *self.claimed.lock(), cell); + let freed = old.as_ref().and_then(|old| old.lock()).is_some_and(|mut queue| { + let busy = !queue.channel_free(); + queue.release_claim(); + busy && queue.channel_free() + }); + if freed { + // A channel nobody prepares into any more is shareable now. + self.shared.channels.notify(); + } + } + + /// Called with the device mutex held after a successful publication: remembers the bound + /// channel for direct publication. + fn remember_direct(&self, registry: &Registry) { + let cell = registry.compute_cell(self.owner); + let mut direct = self.direct.lock(); + let current = match (&*direct, &cell) { + (Some(old), Some(new)) => Arc::ptr_eq(old, new), + (None, None) => true, + _ => false, + }; + if !current { + let old = core::mem::replace(&mut *direct, cell); + drop(direct); + drop(old); + } + } + + /// Publishes a dependency-free compute command from its submit ioctl when + /// its queue is already bound and nothing has to wait, as renders do. Any + /// wait leaves the command, unpublished, to its scheduler. + pub(super) fn try_compute_early(&self, packet: &Arc) -> bool { + if self.compute_backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = match self.compute_direct_packet(packet, &mut deferred) { + Some(result) => result, + None => self.compute_inline(packet, &mut deferred), + }; + if matches!(result, Err(error) if error == render::FALLBACK) && !packet.is_published() { + deferred.finish(); + packet.abandon_publication(); + return false; + } + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } + + fn compute_direct_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Option { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { + return Some(Err(EINVAL)); + }; + let required = match super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) { + Ok(required) => required, + Err(error) => return Some(Err(error)), + }; + self.compute_direct(packet, required, deferred) + } + + /// compute_packet's publication section without its waits: every condition + /// it would sleep or retry for returns FALLBACK instead. + fn compute_inline( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { + return Err(render::FALLBACK); + } + let lease = self.enter_preparation(false)?; + match packet.completion.work_state()?.acquire(crate::g17::job::Engine::Compute) { + Err(error) if error == EBUSY => return Err(render::FALLBACK), + result => { + result?; + } + } + // A lone admission keeps the original latency-sensitive probe. + // Competing publishers otherwise spin on a binding they do not own. + if !self.context.compute_binding_hint() + && self.shared.compute_demand.load(Ordering::Relaxed) > 1 + { + return Err(render::FALLBACK); + } + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() + || self.shared.crashed.load(Ordering::Acquire) + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(render::FALLBACK); + } + firmware.replay_compute(deferred)?; + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + return Err(render::FALLBACK); + } + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { + Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { + Err(render::FALLBACK) + } + result => { + if result.is_ok() { + self.remember_direct(&firmware.queues); + } + result + } + } + } + + /// A scheduled dependency may refer to a previous recovery epoch. Do not + /// acquire consumer capacity until every live source is published now. + fn sources_published(&self, packet: &super::job::Packet) -> Result { + let Some(sources) = packet.completion.sources() else { return Ok(true); }; + sources.check()?; + let state = self.shared.state.lock(); + let firmware = state.as_deref().ok_or(ENODEV)?; + if firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { return Ok(false); } + for source in &sources.sources { + if source.submission.source_status()? { continue; } + if !firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&source.submission, + packet.completion.status(), firmware.recovery.generation())) + { return Ok(false); } + } + Ok(true) } /// Called after taking this packet's publication claim. All waits and work @@ -1326,8 +2068,17 @@ impl Backend { ) -> Result { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let mut recovery_polls = 0; - let mut state_delay = 20i64; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if let Some(result) = self.compute_direct(packet, required, deferred) { + return result; + } + // A bound queue needs none of ensure_compute's admission passes. Try + // without them; the publication section below checks the same state. + // Every retry blocks on the release that ends its wait: retirement + // progress (work states, ring room, replays) and the reopening of + // publication after recovery both signal `progress`. + let mut bound = true; + let mut demand = None; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); let check = || { if packet.cancel_requested() { return Err(ECANCELED); } @@ -1336,24 +2087,37 @@ impl Backend { Ok(()) }; let admission = Admission::new(deadline, &check); + let progress = &self.shared.progress; loop { admission.check()?; + let observed = progress.observe(); + if !self.sources_published(packet)? { + let recovering = self.shared.state.lock().as_deref().is_some_and(|firmware| { + firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 + }); + if recovering { + admission.extend_for_recovery(); + } + self.shared.queue_events(); + self.wait_release(progress, observed, &admission)?; + continue; + } // All DRM dependencies were satisfied before this scheduler callback. // Recheck on every retry: pressure may yield an idle binding between // admission and publication without changing this logical queue. - Backend::ensure_compute(self, scratch.pages(), &admission)?; - admission.check()?; + if !bound { + demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); + Backend::ensure_compute(self, scratch.pages(), &admission)?; + admission.check()?; + } let lease = self.shared.preparations.enter_until(&admission)?; // Work states free as this context's earlier commands retire. - // Wait for one under the packet's admission deadline, as render - // publication does, instead of failing the packet. - match packet.completion.work_state()?.acquire() { + match packet.completion.work_state()?.acquire(crate::g17::job::Engine::Compute) { Err(error) if error == EBUSY => { drop(lease); - admission.check()?; self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); + self.wait_release(progress, observed, &admission)?; continue; } result => { @@ -1376,18 +2140,18 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; + admission.extend_for_recovery(); drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } firmware.replay_compute(deferred)?; - if firmware.queues.compute(self.owner).is_none() { + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + bound = false; drop(state); drop(lease); continue; @@ -1400,29 +2164,27 @@ impl Backend { dependencies.as_slice(), deferred, ); - if matches!(result, Err(EBUSY) | Err(EAGAIN)) - && !packet.is_published() - && (firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0) + if !packet.is_published() + && (matches!(result, Err(EAGAIN)) + || (matches!(result, Err(EBUSY)) + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0))) { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); + // Recovery or backpressure: nothing is firmware-visible. Other + // queues keep submitting and the event worker retires meanwhile. + if firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + admission.extend_for_recovery(); } - recovery_polls += 1; drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } - if matches!(result, Err(EAGAIN)) && !packet.is_published() { - // Backpressure owns no firmware-visible attempt. Other queues - // can submit and the event worker can retire while we sleep. - drop(state); - drop(lease); - self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - continue; + if result.is_ok() { + self.remember_direct(&firmware.queues); } return result; } @@ -1435,7 +2197,7 @@ impl super::Firmware { /// or an allocation-dependent collector. fn finish_stopped_queues(&mut self) { self.stop_render(); - for queue in self + for mut queue in self .queues .compute .iter_mut() @@ -1491,6 +2253,7 @@ impl super::Shared { dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); return; } + self.forget_port(); let firmware = self.state.lock().take(); if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } } @@ -1504,6 +2267,7 @@ impl super::Shared { pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); let _stopping = self.stop.lock(); + self.forget_port(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { @@ -1543,20 +2307,41 @@ impl super::Firmware { deferred: &mut DeferredBatch, now: u64, ) -> Result> { - for queue in self + let mut freed = false; + let mut releasing = false; + for mut queue in self .queues .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.recycle_killed()?; queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } + freed |= busy && queue.channel_free(); + releasing |= queue.pool_release_pending(); + } + freed |= self.queues.observe_pool_releases(&self.init, deferred)? != 0; + let shared = &self.primary.state.shared; + // Edge notifications cover the transitions this driver makes; every + // event pass also rechecks the level, so capacity that appeared through + // another engine (render QIDs, closed logical queues, render-held work + // states) still ends a scheduler wait that has no timeout of its own. + if freed + || (shared.channels.has_waiters() + && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) + { + shared.channels.notify(); + } + // A pool release has no completion event: each observation pass is + // the progress a growing owner waits for. + if releasing || shared.progress.has_waiters() { + shared.progress.notify(); } - self.queues.observe_pool_releases(&self.init, deferred)?; if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) .any(|queue| queue.pool_release_stalled(now)) { return Err(ETIMEDOUT); @@ -1568,6 +2353,11 @@ impl super::Firmware { // retained compute pool stays warm: render and new compute need it more. let cap = if super::freelist::lower_pressure() { 0 + } else if self.primary.state.shared.compute_demand.load(Ordering::Acquire) != 0 { + // A retired channel being requested by another runnable publisher + // is not an unused warm pool. Keep its existing backing through + // handoff; real allocation pressure above still forces reclamation. + super::kick::COMPUTE_CHANNELS } else { COMPUTE_WARM_CAP }; @@ -1594,12 +2384,19 @@ impl Registry { writeln!(f, "qos_ids used={} total=126", self.qos_ids.used())?; writeln!(f, "metrics_ids used={} total={}", self.metrics.used(), super::fw::buffer::METRICS_SLOTS)?; self.render.write_capacity(f)?; + let now = now_ns(); + let waits = || self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) + .filter_map(|queue| queue.kill_waiting(now)); + compute_exit::write_kill_waits(f, waits().count(), waits().max().unwrap_or(0))?; + compute_exit::write_render_kills(f, self.teardown.kill_waits(now).count(), + self.teardown.kill_waits(now).max().unwrap_or(0))?; writeln!(f, "compute_slots used={} total={} reusable={} building={}", self.compute.iter().flatten().count(), self.compute.len(), self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; - writeln!(f, "compute_channels used={} total={}", - self.compute.iter().flatten().count(), super::kick::COMPUTE_CHANNELS)?; + writeln!(f, "compute_channels used={} fresh_fits={}", + self.compute.iter().flatten().count(), + u8::from(self.qids.compute_fits(self.admission.queues() as usize)))?; writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { queue.write_capacity(f)?; @@ -1609,13 +2406,42 @@ impl Registry { } impl Backend { - pub(super) fn wait_capacity(&self, started: u64) -> Result { - let check = || self.admission_open(); - let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); - self.wait_compute_capacity(&admission) + /// Compute admission without a channel: give back what pressure allows, then block until a + /// channel is released after `observed` (sampled before the failed check). + fn wait_channels(&self, observed: u64, admission: &Admission<'_>, reclaim_backing: bool) -> Result { + self.reclaim_capacity(admission, reclaim_backing)?; + self.wait_release(&self.shared.channels, observed, admission) + } + + /// Blocks until `domain` is signalled after `observed`, this packet's admission ends, or one + /// admission slice passes: cancellation and logical close have no release to wait for. + fn wait_release( + &self, + domain: &super::availability::Availability, + observed: u64, + admission: &Admission<'_>, + ) -> Result { + use kernel::dma_fence::RawDmaFence; + let slice = admission.slice_ms()?; + if let Some(fence) = domain.wait(observed)? { + // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. + let waited = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(slice) as isize, + ) + }; + if waited < 0 { + return Err(Error::from_errno(waited as i32)); + } + } + admission.check() } - fn wait_compute_capacity(&self, admission: &Admission<'_>) -> Result { + /// Reclaim steps of a capacity wait that never sleep: observe consumed pool releases and, + /// under backing pressure, give idle pools back. + fn reclaim_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { admission.check()?; let mut deferred = DeferredBatch::publication()?; let result: Result = (|| { @@ -1630,14 +2456,15 @@ impl Backend { if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { return Ok(()); } - for queue in firmware.queues.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { - admission.check()?; - queue.share_idle()?; - } // Consumption, not publication or an elapsed timeout, frees backing. firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; - firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; - if !firmware.render_control_backpressured() { + // Pool eviction addresses backing allocation pressure, not channel + // occupancy. Normal idle expiry and the retained-pool cap still run + // in the event worker for either kind of admission wait. + if reclaim_backing { + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + } + if reclaim_backing && !firmware.render_control_backpressured() { let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } } @@ -1649,8 +2476,6 @@ impl Backend { Err(error) if error == EAGAIN => {} result => result?, } - admission.check()?; - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); admission.check() } } @@ -1669,12 +2494,17 @@ mod shared_capacity_tests { ) .unwrap(); let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - // Physical compute channels are device-owned and bounded independently - // of how many logical clients have queued unsignalled work. - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + let mut slots = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..crate::mmu::MAX_EXECUTION_CONTEXTS { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // With every execution context open, compute keeps only the channels + // that leave each logical queue a dedicated render pair. + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { registry.reserve_compute(owner).unwrap(); } assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); + assert!(registry.qids.render_fits(slots.len())); for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); @@ -1695,9 +2525,6 @@ mod shared_capacity_tests { GFP_KERNEL, ) .unwrap(); - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { - registry.reserve_compute(owner).unwrap(); - } let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); for owner in 1..=u64::from(limit) { @@ -1726,11 +2553,31 @@ mod shared_capacity_tests { leases.push(lease, GFP_KERNEL).unwrap(); } } - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { registry.reserve_compute(owner).unwrap(); } assert_eq!(leases.len(), 36); } + + #[test] + fn compute_grows_with_free_qids_and_narrows_queue_admission() { + let mut registry = KBox::init(Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), GFP_KERNEL).unwrap(); + let mut slots = KVec::with_capacity(20, GFP_KERNEL).unwrap(); + for _ in 0..20 { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // Sixteen open compute clients each keep their own channel. + for owner in 1..=16u64 { + registry.reserve_compute(owner).unwrap(); + } + // Growth stops where open queues plus headroom would lose their pairs. + let mut owner = 17; + while registry.reserve_compute(owner).is_ok() { + owner += 1; + } + assert!(registry.qids.render_fits(slots.len() + 24)); + assert!(!registry.qids.render_fits(crate::mmu::MAX_EXECUTION_CONTEXTS as usize)); + } } impl Backend { diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 7d690470cd7b5..bf0f989f042a5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -13,7 +13,14 @@ use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { if packet.take_backlog() { - self.backlog.fetch_sub(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_sub(1, Ordering::AcqRel); + } + } + + pub(super) fn backlog_of(&self, packet: &Packet) -> &core::sync::atomic::AtomicU32 { + match packet.engine() { + Engine::Render => &self.backlog, + Engine::Compute => &self.compute_backlog, } } @@ -48,6 +55,7 @@ impl Backend { drop(state); drop(binding); self.finish_uninstalled_compute(retired)?; + self.shared.channels.notify(); self.shared.queue_events(); return Ok(()); } @@ -85,6 +93,7 @@ impl Backend { // A builder may own only an off-lock reservation when close arrives. // Record intent before scanning queues; every later install/rearm checks it. self.closed.store(true, Ordering::Release); + self.hold_claim(None); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -99,12 +108,63 @@ impl Backend { firmware.queues.release_render(self.owner)?; firmware.queues.admission.release_owner(self.owner); drop(state); + // Its idle channel, or its slot and never-published QID, is free. + self.shared.channels.notify(); self.shared.queue_events(); Ok(()) } } impl job::Backend for Backend { + fn input_sources(&self, inputs: &[crate::file::SyncItem]) + -> Option> + { + use crate::g17::{fence, sync_input}; + let mut heads = KVec::new(); + let mut sources: KVec = KVec::new(); + let mut publication_waits = KVec::new(); + for input in inputs.iter().filter_map(|input| input.fence.as_ref()) { + let status = job::fence_status(input); + if status > 0 { continue; } + if status < 0 { return None; } + let leaves = sync_input::collect(input)?; + for leaf in &leaves { + if leaf.status > 0 { continue; } + if leaf.status < 0 { return None; } + // SAFETY: Collection owns each leaf until this lookup finishes. + let submission = unsafe { fence::exported_source_raw(leaf.fence) }?; + if submission.source_status().ok()? { continue; } + if !submission.same_vm(self.context.status()) { return None; } + if sources.iter().any(|known| Arc::ptr_eq(&known.submission, &submission)) { + continue; + } + if sources.len() == crate::g17::fw::kick::KICK_BARRIERS_MAX { return None; } + let published = { + let state = self.shared.state.lock(); + let firmware = state.as_deref()?; + if self.shared.crashed.load(Ordering::Acquire) || firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init).ok()? != 0 + { return None; } + firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&submission, self.context.status(), + firmware.recovery.generation())) + }; + if !published { + if submission.source_status().ok()? { continue; } + // The existing scheduler fence becomes ready after actual + // publication, not GPU completion. No binding loan or + // consumer hardware capacity is acquired while waiting. + publication_waits.push(submission.ready()?, GFP_KERNEL).ok()?; + } + sources.push(sync_input::Source { submission }, GFP_KERNEL).ok()?; + } + heads.push(input.clone(), GFP_KERNEL).ok()?; + } + if sources.is_empty() { return None; } + sync_input::Sources::new(heads, sources, publication_waits).ok() + } + fn feed(&self) -> Option> { Some(self.shared.feed.clone()) } @@ -156,8 +216,13 @@ impl job::Backend for Backend { /// Firmware kick parents order the same engine. Explicit cross-engine and /// input dependencies were already attached to the DRM scheduled job. - fn prepare(&self, _packet: &Packet) -> Option { - None + /// Compute and render admission wait for their shared capacity here, as a + /// fence. + fn prepare(&self, packet: &Arc) -> Option { + match packet.engine() { + Engine::Compute => self.prepare_compute(packet), + Engine::Render => self.admit_render(packet), + } } fn publish(&self, packet: Arc) -> Result { @@ -185,6 +250,9 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -213,6 +281,9 @@ impl job::Backend for Backend { fn cancel(&self, packet: Arc) { self.release_backlog(&packet); + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; @@ -226,6 +297,16 @@ impl job::Backend for Backend { } } + fn long_job_beside(&self, _packet: &Arc) -> bool { + let state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref() else { + return false; + }; + firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|other| other.owner() != Some(self.owner) && other.long_job_running()) + } + fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool { if packet.engine() == Engine::Compute { return self.defer_compute_timeout(packet, renewals); @@ -274,11 +355,14 @@ impl job::Backend for Backend { } fn try_early(&self, packet: &Arc) -> bool { - self.try_render_early(packet) + match packet.engine() { + Engine::Render => self.try_render_early(packet), + Engine::Compute => self.try_compute_early(packet), + } } fn count_backlog(&self, packet: &Packet) { packet.count_backlog(); - self.backlog.fetch_add(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_add(1, Ordering::AcqRel); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index c407d9576e9c4..4f519743aed53 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,11 +20,10 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute(self.owner) - .filter(|queue| queue.can_classify_timeout(packet)) else { - return Ok(Decision::Own); - }; - let Some(own) = queue.timeout_head() else { + // Each queue is locked on its own: release this one before visiting the others. + let Some(own) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) + .and_then(|queue| queue.timeout_head()) else { return Ok(Decision::Own); }; let computes = firmware.queues.compute.iter().flatten() @@ -64,7 +63,7 @@ impl Backend { return Ok(None); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute_mut(self.owner) else { + let Some(mut queue) = firmware.queues.compute_mut(self.owner) else { return Ok(None); }; if !queue.owns_packet(packet) { @@ -91,7 +90,7 @@ impl Backend { return Ok(()); }; firmware.scan_compute(None, &mut deferred)?; - if let Some(queue) = firmware.queues.compute_mut(self.owner) { + if let Some(mut queue) = firmware.queues.compute_mut(self.owner) { queue.cancel(packet, &mut |event| deferred.push(event))?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index d71f410c76fb4..32b9acef94c0f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -3,7 +3,7 @@ //! Ordered republication of witnessed, unexecuted compute commands. use super::DeferredBatch; -use crate::g17::{completion::Deferred, qos, recovery, Firmware}; +use crate::g17::{completion::Deferred, recovery, Firmware}; use kernel::prelude::*; impl Firmware { @@ -31,12 +31,12 @@ impl Firmware { return Ok(()); } deferred.reserve_replays(commands)?; - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { queue.prepare_replays( state, - |owner| self.queues.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -46,20 +46,14 @@ impl Firmware { if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { return Ok(()); } - let Some(queue) = self.queues.compute[index] + let Some((packet, owner)) = self.queues.compute[index] .as_ref() .and_then(|entry| entry.queue.as_deref()) + .and_then(|queue| Some((queue.replay_front()?, queue.owner()))) else { break; }; - let Some(packet) = queue.replay_front() else { - break; - }; - let owner = queue.owner(); - let outer_ready = self.init.work_ready( - packet.context.policy().priority(), - crate::g17::fw::queue::DataMaster::Compute, - )?; + let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); @@ -90,7 +84,7 @@ impl Firmware { let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); - let queue = self.queues.compute[index] + let mut queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; @@ -110,6 +104,8 @@ impl Firmware { } } self.queues.replay_outer_wait = None; + // Publication behind these replays may proceed. + self.primary.state.shared.progress.notify(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4fd6e56741986..3e779bba4fbbf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -15,6 +15,9 @@ use kernel::{prelude::*, sync::Arc}; /// Wake interval while a tiler heap may still shrink: the native check period. const SHRINK_WAKE_NS: u64 = 1_000_000_000; +/// Longest interval between two event passes that run every housekeeping step while +/// completion records alone keep the worker busy. +const HOUSEKEEPING_NS: u64 = 1_000_000; struct Service<'a> { firmware: &'a mut Firmware, @@ -35,16 +38,19 @@ impl event::Transport for Service<'_> { } fn statistics(&mut self, role: Role) { let firmware = &mut *self.firmware; + firmware.observe_statistics_power(); firmware .rings .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); + firmware.observe_statistics_power(); } fn drain( &mut self, role: Role, mut consume: impl FnMut(&mut Self, Event) -> Result, ) -> Result { - let mut events = channel::Events::new(role); + let mut events = + channel::Events::new(role, self.firmware.primary.state.shared.variant.graph); while let Some(event) = events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? { @@ -72,6 +78,15 @@ impl event::Queues for Service<'_> { fn context_killed(&mut self, cookie: u64) { self.firmware.queues.acknowledge_compute_exit(cookie); } + fn compute_fault(&mut self, qid: u8, stamp: u32) { + for entry in self.firmware.queues.compute.iter_mut().flatten() { + if let Some(mut queue) = entry.queue.as_deref_mut() { + if queue.qid() == qid { + queue.note_fault(stamp); + } + } + } + } fn flush_tvb_reply(&mut self) -> Result { self.firmware.flush_tvb_reply() } @@ -131,8 +146,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entries(&self) -> Result { - self.firmware.init.entries() + fn entry(&self, index: usize) -> Result<(u32, u32)> { + self.firmware.init.entry(index) } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -203,6 +218,27 @@ impl recovery::Queues for Service<'_> { } self.firmware.queues.classify_render(class) } + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> recovery::Interrupted { + let render = self.firmware.queues.note_render_interrupted(qid, stamp); + if render != recovery::Interrupted::None { + return render; + } + self.firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.qid() == qid) + .map_or(recovery::Interrupted::None, |queue| { + queue.note_interrupted(stamp) + }) + } + fn report_blamed_render(&self, qid: u8) { + self.firmware + .queues + .report_render_recovery(qid, &self.firmware.port); + } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self .firmware @@ -225,19 +261,17 @@ impl recovery::Queues for Service<'_> { impl Firmware { fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { let state = recovery::Memory::recovery_state(&self.init)?; - let view = crate::g17::qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.release_witnessed( - state, - |owner| registry.accounting.complete(&view, &[owner]), - &mut |event| deferred.push(event), - )?; + queue.release_witnessed(state, |owner| port.qos_complete(&[owner]), &mut |event| { + deferred.push(event) + })?; } Ok(()) } @@ -266,7 +300,11 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { - if self.recovery.restart_limit(super::now_ns()) { return Err(ETIMEDOUT); } + self.rings.invalidate_statistics(); + if self.recovery.restart_limit(super::now_ns()) { + pr_warn!("G17: restart storm limit reached\n"); + return Err(ETIMEDOUT); + } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, @@ -276,7 +314,15 @@ impl Firmware { }; let result = recovery::service(&mut service, || { shared.crashed.load(core::sync::atomic::Ordering::Acquire) - })?; + }); + // Publishers without the device mutex stamp their tickets with this + // mirror; they rejoin only after preparation admission reopens. + service + .firmware + .port + .set_generation(service.firmware.recovery.generation()); + service.firmware.rings.invalidate_statistics(); + let result = result?; if let Some(error) = result.target_error { dev_warn!( shared.dev.as_ref(), @@ -319,8 +365,10 @@ impl crate::g17::Shared { }; deferred }; + crate::g17::trace::event(0); let result = self.runtime_event_turn(&mut deferred); deferred.finish(); + crate::g17::trace::event(2); self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); @@ -333,6 +381,10 @@ impl crate::g17::Shared { } fn runtime_event_turn(self: &Arc, deferred: &mut DeferredBatch) -> Result { + // Taken before consuming records: a request made during this pass gets the next one. + let requested = self + .housekeeping + .swap(false, core::sync::atomic::Ordering::AcqRel); let (recover, masks) = { let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -348,8 +400,10 @@ impl crate::g17::Shared { (recover, masks) }; deferred.finish(); + crate::g17::trace::event(1); if recover { - self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; + self.preparations + .wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } let mut state = self.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { @@ -392,19 +446,44 @@ impl crate::g17::Shared { .any(|queue| queue.in_flight()); let in_flight = compute || firmware.queues.render_in_flight(); let idle = firmware.idle_check(now, in_flight)?; - firmware.reclaim_drained_renders(deferred)?; - firmware.service_compute_exit(deferred)?; - let pools = firmware.service_compute_pools(deferred, now)?; + // Expiries, drained pairs, exits, pool and heap trimming and teardown give back what + // retirement freed; a completion needs none of them to retire or to signal. While work + // is in flight and nothing waits for capacity, a pass for firmware records alone skips + // them. Any requested pass, recovery, teardown, a waiter, the last retirement and, at + // the latest, the first pass after HOUSEKEEPING_NS run them all. + let housekeeping = requested + || recover + || !in_flight + || firmware.recovery.pending() + || !firmware.queues.teardown.is_empty() + || self.channels.has_waiters() + || self.progress.has_waiters() + || self.queues.has_waiters() + || self.renders.has_waiters() + || now.saturating_sub( + self.housekeeping_ns + .load(core::sync::atomic::Ordering::Relaxed), + ) >= HOUSEKEEPING_NS; + let mut lingers = None; + let mut pools = None; let mut shrinking = false; - if !firmware.recovery.pending() { - firmware - .queues - .vacate_evicted_render_pools(&firmware.init, deferred)?; - shrinking = firmware - .queues - .shrink_render_heaps(&firmware.init, now, deferred); - } - { + if housekeeping { + self.housekeeping_ns + .store(now, core::sync::atomic::Ordering::Relaxed); + if !firmware.recovery.pending() { + lingers = firmware.expire_render_lingers(now)?; + } + firmware.reclaim_drained_renders(deferred)?; + firmware.service_compute_exit(deferred)?; + pools = firmware.service_compute_pools(deferred, now)?; + if !firmware.recovery.pending() { + firmware + .queues + .vacate_evicted_render_pools(&firmware.init, deferred)?; + shrinking = firmware + .queues + .shrink_render_heaps(&firmware.init, now, deferred); + } let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; if !reclaim.is_empty() { @@ -434,11 +513,14 @@ impl crate::g17::Shared { } else { idle }; - // A retained compute pool gives its backing back when it expires. - let idle = match (idle, pools) { - (Some(delay), Some(expiry)) => Some(delay.min(expiry)), - (delay, expiry) => delay.or(expiry), - }; + // A retained compute pool gives its backing back when it expires, and + // so does an idle render free-list registration. + let idle = [pools, lingers] + .into_iter() + .fold(idle, |idle, expiry| match (idle, expiry) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }); if let Some(delay) = idle { self.queue_idle(delay); } @@ -453,13 +535,18 @@ impl crate::g17::Shared { { // A full bounded pass must not depend on a fresh notification for // records that arrived while it was consuming the same ring. - self.queue_events(); + self.queue_event_records(); } let grow_ready = firmware.render_grow_ready(); drop(state); if grow_ready { self.queue_grow(); } + // Firmware ring room, consumed releases and teardown steps have no + // event of their own: recheck render publishers' conditions every pass. + if self.renders.has_waiters() { + self.renders.notify(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 59db76b9f109a..6657f83e9e738 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -6,6 +6,7 @@ mod exit; mod dependencies; +mod direct; mod grow; mod observe; mod pools; @@ -16,6 +17,7 @@ mod settle; mod submit; pub(super) mod teardown; +pub(super) use submit::FALLBACK; use super::{DeferredBatch, Registry}; use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; use crate::hw::t8140::queues::RENDER_SLOTS; @@ -74,6 +76,9 @@ pub(super) struct RenderState { pools: pools::Pools, grow: grow::Grow, retired: KVec, + /// Graphs of released contexts the firmware may still walk (see `detach_context`): kept + /// until the firmware stops, when this state is dropped. + kept_graphs: KVec, next_pm_generation: u64, } @@ -100,6 +105,7 @@ impl RenderState { pools: pools::Pools::new()?, grow: grow::Grow::new()?, retired: KVec::with_capacity(128, GFP_KERNEL)?, + kept_graphs: KVec::new(), next_pm_generation: 1, }) } @@ -298,6 +304,7 @@ impl Registry { if let Some(pair) = entry.pair.as_ref() { self.buffers.unreserve_owner(pair.buffer_id())?; } + self.note_build_room(); } return Ok(Some(UnusedRenderSpare { _entry: entry, @@ -325,6 +332,28 @@ impl Registry { Ok(None) } + /// The installed pair in `slot` belongs to `owner` and its context, is + /// neither borrowed nor settling a failure, and its pool has backing: the + /// state in which building the pair would do nothing. + pub(super) fn render_ready(&self, slot: u8, owner: u64, context: &Arc) -> Result { + let Ok(entry) = self.render.entry(slot) else { + return Ok(false); + }; + if entry.pending_error.is_some() + || entry.pending_timeout + || entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.parked.is_some() + { + return Ok(false); + } + let Some(pair) = entry.pair.as_ref() else { + return Ok(false); + }; + Ok(pair.matches(owner, context)? && !pair.pool().vacant()) + } + fn has_render_ids(&self, owner: u64) -> bool { self.render .entries @@ -333,11 +362,25 @@ impl Registry { .any(|(slot, entry)| entry.is_some() && self.admission.owns_slot(owner, slot)) } + /// Tell admission whether another render pair can still be built. + pub(super) fn note_build_room(&self) { + self.admission.note_build_room(self.qids.render_pair_fits()); + } + pub(super) fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { return Err(EINVAL); } - if self.render.entries[usize::from(slot)].is_none() { + if self.render.entries[usize::from(slot)].is_some() { + return Ok(()); + } + let built = self.build_render_ids(slot, owner); + self.note_build_room(); + built + } + + fn build_render_ids(&mut self, slot: u8, owner: u64) -> Result { + { let identity = self.identities.reserve(2)?.start; let tiling = self.qids.reserve(identity, DataMaster::Tiling, None)?; let fragment = match self.qids.reserve(identity + 1, DataMaster::Fragment, None) { @@ -381,6 +424,13 @@ impl Registry { /// ensure_render_ids cancels a first ID if the second cannot be reserved; /// validation, identity overflow, and cancellation errors still propagate. fn prepare_render_ids(&mut self, slot: u8, owner: u64, optional: bool) -> Result { + // A full queue table refuses a sibling without scanning it on every preparation. + if optional + && self.render.entries.get(usize::from(slot)).is_some_and(Option::is_none) + && !self.qids.render_pair_fits() + { + return Ok(false); + } match self.ensure_render_ids(slot, owner) { Ok(()) => Ok(true), Err(error) if optional && error == ENOSPC => Ok(false), @@ -507,27 +557,16 @@ impl crate::g17::queue::render::Host for Host<'_> { core::sync::atomic::fence(Ordering::Acquire); Ok(stamps) } - fn qos_publish( + fn qos_publish_pair( &mut self, - owner: crate::g17::qos::Owner, + owners: [crate::g17::qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result { - use crate::g17::qos; - self.firmware.queues.accounting.publish( - &qos::View::new(self.firmware.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + ) -> Result<[crate::g17::qos::Publication; 2]> { + self.firmware.port.qos_publish_pair(owners, scheduler, policy) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { - use crate::g17::qos; - self.firmware - .queues - .accounting - .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) + self.firmware.port.qos_cancel(publication) } /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration @@ -558,7 +597,7 @@ impl crate::g17::queue::render::Host for Host<'_> { self.firmware.queues.qids.publish(id) } fn next_publication(&mut self) -> Result { - self.firmware.queues.qids.next_publication() + Ok(self.firmware.port.next_publication()) } fn publish_outer_pair( &mut self, @@ -599,6 +638,9 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); + crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { + self.firmware.bridge.regs.g17_core_power() + }); let result = self .firmware .primary @@ -618,13 +660,20 @@ impl crate::g17::queue::render::Host for Host<'_> { generation: u64, buffer_id: u8, ) -> Result { - use crate::g17::qos; let queues = &mut self.firmware.queues; - let next = queues.render.pools.completed(pool_id, generation)?; + // Each final release is a control record the firmware processes before + // it returns to the work rings. While other pairs keep it busy, a + // registration left idle lingers instead, and the owner's next render + // reuses it. On an otherwise idle device the release is cheaper than + // reusing a registration that went idle, so it is sent at once. + // The pair retiring here is lent out of its entry meanwhile. + let linger = queues.render.entries.iter().flatten().any(|entry| { + entry.pair.as_ref().is_some_and(|pair| pair.published_in_flight()) + }); + let next = queues.render.pools.completed(pool_id, generation, linger)?; queues.buffers.can_release(buffer_id)?; - let view = qos::View::new(self.firmware.init.qos()?)?; if let Some(owners) = owners { - queues.accounting.complete(&view, &owners)?; + self.firmware.port.qos_complete(&owners)?; } queues.render.pools.commit_completed(pool_id, next); queues.buffers.release(buffer_id) @@ -657,6 +706,30 @@ impl crate::g17::Firmware { result } + /// Event-worker turn: ends free-list lingers that elapsed or whose owner + /// closed or moved on, publishes their releases, and returns the delay to + /// the next expiry, which the worker turns into a timed wake. + pub(in crate::g17) fn expire_render_lingers(&mut self, now: u64) -> Result> { + let render = &mut self.queues.render; + let entries = &render.entries; + let (ended, next) = render.pools.expire_lingers(now, |owner| { + entries.iter().flatten().any(|entry| { + !entry.closed + && entry + .pool + .as_ref() + .is_some_and(|pool| pools::Pools::owner(pool) == *owner) + }) + }); + if ended { + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + Ok(next) + } + fn release_render_lists(&mut self) -> Result { if self.queues.render.grow.reply.is_some() { return Err(EAGAIN); @@ -725,6 +798,19 @@ impl crate::g17::Firmware { { continue; } + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if let Some(pair) = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + { + let qids = pair.qids(); + let stamps = [ + self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), + self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), + ]; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + pair.note_stamp_wait(stamps, state, crate::g17::now_ns())?; + } self.retire_render_slot(slot, None, deferred)?; self.queues.render.collect_retired(slot)?; self.settle_render_terminal(slot, deferred)?; @@ -783,6 +869,22 @@ impl Registry { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_spared()) } + /// Marks the interrupted group of the pair that owns `qid`. + pub(in crate::g17) fn note_render_interrupted( + &self, + qid: u8, + stamp: u32, + ) -> crate::g17::recovery::Interrupted { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .map_or(crate::g17::recovery::Interrupted::None, |pair| { + pair.note_interrupted(stamp) + }) + } pub(in crate::g17) fn visit_render_owners( &self, qid: u8, @@ -968,10 +1070,10 @@ impl Registry { |pair| pair.published_in_flight(), ); active - && entry.reservation.ids.into_iter().all(|id| { - let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] & (1u64 << (qid % 64)) != 0 - }) + && (crate::g17::queue::render::completion_noted( + entry.reservation.ids.map(kick::Id::qid), + entry.event_masks, + ) || entry.pair.as_ref().is_some_and(|pair| pair.stamp_waiting())) }) } pub(in crate::g17) fn defer_render_timeout( @@ -1056,6 +1158,8 @@ impl Registry { impl RenderState { pub(super) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { writeln!(f, "retired_render_bindings used={} total={}", self.retired.len(), self.retired.capacity())?; + writeln!(f, "firmware_linked_render_graphs kept={} leaked={}", self.kept_graphs.len(), + teardown::LEAKED_GRAPHS.load(core::sync::atomic::Ordering::Relaxed))?; self.pools.write_capacity(f)?; for (slot, entry) in self.entries.iter().enumerate() { if let Some(context) = entry.as_ref().and_then(|entry| entry.context.as_ref()) { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index e3cb174721aaa..277534328406e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -14,8 +14,8 @@ use kernel::prelude::*; const RETRY_LIMIT: u16 = 1000; pub(super) struct Grow { - requests: KVec, - active: Option, + pub(super) requests: KVec, + pub(super) active: Option, pub(super) reply: Option<(TvbGrowRequest, bool, u16)>, release_attempts: u16, } @@ -280,7 +280,7 @@ impl crate::g17::Shared { return Err(EIO); } firmware.queues.render.grow.active = None; - self.changed.notify_all(); + self.note_changed(); if !lease.is_current() || request.halt_count != firmware.recovery.generation() { continue; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 4ede20d316c64..57543be54e835 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -1,21 +1,22 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Event-driven render snapshots outside the device mutex. The preparation -//! epoch and exact pair owner survive every read, cancellation and retry. +//! Event-driven render snapshots. A pair's completion memory is read and the +//! pair retired in one device-mutex hold, inside the preparation epoch. use super::super::DeferredBatch; -use crate::g17::queue::render::Pair; use core::sync::atomic::Ordering; use kernel::{prelude::*, sync::Arc}; impl crate::g17::Firmware { - fn take_render_observation(&mut self, start: usize) -> Result)>> { + /// The first slot from `start` whose pair has news to observe: published work in flight, the + /// fragment queue's notification seen (`completion_noted`), and an observation the pair accepts. + fn next_render_observation(&mut self, start: usize) -> Result> { if self.recovery.pending() { return Ok(None); } let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; for slot in start..self.queues.render.entries.len() { - let Some(entry) = self.queues.render.entries[slot].as_mut() else { + let Some(entry) = self.queues.render.entries[slot].as_ref() else { continue; }; if entry.preparing.is_some() || entry.growing { @@ -26,9 +27,8 @@ impl crate::g17::Firmware { }; let qids = pair.qids(); if !pair.published_in_flight() - || !qids - .into_iter() - .all(|qid| entry.event_masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) + || (!crate::g17::queue::render::completion_noted(qids, entry.event_masks) + && !pair.stamp_waiting()) { continue; } @@ -37,35 +37,22 @@ impl crate::g17::Firmware { self.init.stamp(qids[1])?.load(Ordering::Relaxed), ]; core::sync::atomic::fence(Ordering::Acquire); - let Some(packet) = pair.observation_packet(entry.event_masks, stamps, state)? else { - continue; - }; - entry.borrowed_active = pair.published_in_flight(); - entry.borrowed_dependencies = Some(pair.dependency_snapshot()); - entry.borrowed_switch = pair.switch_in_flight(); - entry.preparing = Some(packet); - return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); + if pair.observation_packet(entry.event_masks, stamps, state)?.is_some() { + return Ok(Some(slot)); + } + // The stamps gate this pair out. If its front executed and was consumed anyway, the + // stamps may never move: after the bound, observe it and retire it on its record. + let pair = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .ok_or(EIO)?; + if pair.note_stamp_wait(stamps, state, crate::g17::now_ns())? { + return Ok(Some(slot)); + } } Ok(None) } - fn return_render_observation(&mut self, slot: usize, pair: &mut Option>) -> Result { - let returning = pair.as_ref().ok_or(EINVAL)?; - let entry = self.queues.render.entry_mut(slot as u8)?; - if entry.pair.is_some() - || entry.preparing.is_none() - || returning.slot() as usize != slot - || returning.qids() != entry.reservation.ids.map(crate::g17::kick::Id::qid) - { - return Err(EIO); - } - entry.pair = pair.take(); - entry.borrowed_dependencies = None; - entry.borrowed_switch = None; - entry.preparing = None; - Ok(()) - } - pub(super) fn settle_render_terminal( &mut self, slot: usize, @@ -122,57 +109,36 @@ impl crate::g17::Shared { Err(error) if error == EAGAIN || error == ENODEV => break, result => result?, }; - let loan = { - let mut state = self.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - if !epoch.is_current() || self.crashed.load(Ordering::Acquire) { - break; - } - firmware.take_render_observation(cursor)? + // Selecting, reading and retiring the front share one hold: a + // separate hold for the read would let publishers and the + // retirement contend twice per observed pair. + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); }; - let Some((slot, pair)) = loan else { + if !epoch.is_current() + || firmware.recovery.pending() + || self.crashed.load(Ordering::Acquire) + { + break; + } + let Some(slot) = firmware.next_render_observation(cursor)? else { break; }; if slot != cursor { remaining = crate::hw::t8140::queues::RENDER_DEPTH; retries = 0; } - let observed = pair.observe_front(); - let mut pair = Some(pair); - let mut state = self.state.lock(); - // Return installed ownership before examining read errors or epoch. - let restored = (*state) - .as_deref_mut() - .ok_or(ENODEV) - .and_then(|firmware| firmware.return_render_observation(slot, &mut pair)); - if let Err(error) = restored { - // A failed invariant must not drop firmware-owned storage. Join - // the other preparers only after releasing our own epoch, and - // keep this detached owner until both processors have stopped. - drop(state); - drop(epoch); - self.lose_device_quietly(); - self.stop_runtime(); - if let Some(mut pair) = pair { - pair.stopped(&mut |event| { - event.finish(); - Ok(()) - })?; - } - return Err(error); - } - let firmware = (*state).as_deref_mut().ok_or(EIO)?; - self.changed.notify_all(); + let observed = firmware.queues.render.entries[slot] + .as_ref() + .and_then(|entry| entry.pair.as_ref()) + .ok_or(EIO)? + .observe_front(); + // Publishers waiting for render capacity recheck once this hold ends. + self.note_changed(); if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { - self.queue_events(); - } - if !epoch.is_current() - || firmware.recovery.pending() - || self.crashed.load(Ordering::Acquire) - { - break; + // Observation and the growth check run on every pass. + self.queue_event_records(); } let result = (|| { let state = crate::g17::recovery::Memory::recovery_state(&firmware.init)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 4713eb5b6b152..1e3354d90dbfb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -6,7 +6,7 @@ use crate::g17::{ channel::Rings, - freelist::{Lease, Owner, RenderPool}, + freelist::{Lease, Owner, RenderPool, LINGER_NS}, fw::{ channels::{ControlRecord, FreeListCompletion}, initdata::PagePoolDescriptor, @@ -49,7 +49,7 @@ impl Pools { Ok(slot) } - fn owner(pool: &RenderPool) -> Owner { + pub(super) fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), epoch: pool.epoch(), @@ -154,12 +154,12 @@ impl Pools { } /// Preflight a paired retirement before QoS or PB reference accounting. - pub(super) fn completed(&self, id: u16, generation: u64) -> Result { + pub(super) fn completed(&self, id: u16, generation: u64, linger: bool) -> Result { let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; if next.generation() != generation { return Err(EIO); } - next.complete_render()?; + next.complete_render(linger)?; Ok(next) } @@ -169,6 +169,32 @@ impl Pools { self.leases[usize::from(id)] = Some(next); } + /// Ends the linger of every idle registration that `keep` (an open owner + /// still bound to the pool) does not retain or whose linger has elapsed. + /// Returns whether a release became pending, and the next expiry delay. + pub(super) fn expire_lingers( + &mut self, + now: u64, + keep: impl Fn(&Owner) -> bool, + ) -> (bool, Option) { + let mut ended = false; + let mut next: Option = None; + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { + let Some(since) = lease.lingering() else { + continue; + }; + let age = now.saturating_sub(since); + if age >= LINGER_NS || !keep(&lease.owner()) { + lease.end_linger(); + ended = true; + } else { + let delay = LINGER_NS - age; + next = Some(next.map_or(delay, |next| next.min(delay))); + } + } + (ended, next) + } + pub(super) fn pending(&self) -> bool { self.leases .iter() @@ -186,7 +212,7 @@ impl Pools { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { lease.observe_releases(consumer, producer)?; } Ok(()) @@ -199,7 +225,7 @@ impl Pools { init: &InitData, mut announce: impl FnMut() -> Result, ) -> Result { - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { let Some(release) = lease.pending_release() else { continue; }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index e2034c3176595..62ca7895a86f8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -426,7 +426,7 @@ impl super::super::Backend { } firmware.render_grow_ready() }; - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -548,7 +548,7 @@ impl super::super::Backend { .release(id)?; } } - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -593,6 +593,10 @@ impl super::super::Backend { pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { self.build_render_pair(slot, packet, false, sleep)?; + // An owner has exactly one pair (kick.rs asserts it): no sibling to warm. + if crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1 { + return Ok(()); + } let sibling = { let state = self.shared.state.lock(); (*state) @@ -711,7 +715,7 @@ impl super::super::Backend { .ok_or(EIO)? .release_owner(); } - self.shared.changed.notify_all(); + self.shared.note_changed(); let published = (|| { let entry = firmware.queues.render.entry(slot)?; if entry.pending_error.is_some() || entry.pending_timeout { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 24c724cb32470..71f0bfc052696 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -79,6 +79,20 @@ impl crate::g17::Firmware { } let generation = self.recovery.generation(); let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if error == ETIMEDOUT { + let qids = pair.qids(); + let stamps = [ + self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), + self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), + ]; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + if let Some(pair) = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + { + pair.expire_stamp_wait(packet, stamps, state)?; + } + } let progressed = self.with_render(slot, |pair, host| { pair.retire(None, generation, state, host, &mut |event| { deferred.push(event) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index b77fc13b7ed2f..970035547dfee 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -8,22 +8,18 @@ use crate::g17::{ queue::admission::{RenderLease, Request}, }; use core::sync::atomic::Ordering; -use kernel::{ - prelude::*, - sync::Arc, - time::{delay::fsleep, Delta}, -}; +use kernel::{prelude::*, sync::Arc}; /// Internal result of a publication attempted from the submitting ioctl that /// met contention before anything became visible: the scheduler publishes the /// command instead. Never returned to userspace. -pub(super) const FALLBACK: Error = EINPROGRESS; +pub(in crate::g17::runtime) const FALLBACK: Error = EINPROGRESS; impl Backend { /// The submitting ioctl holds the logical queue's mutex and never sleeps /// for contention or a closed preparation gate: it returns FALLBACK, and /// the scheduler's worker publishes the command and owns the waits. - pub(super) fn enter_preparation(&self, sleep: bool) -> Result { + pub(in crate::g17::runtime) fn enter_preparation(&self, sleep: bool) -> Result { if sleep { return self.shared.preparations.enter(false); } @@ -48,21 +44,32 @@ impl Backend { /// Reserve identities before attaching a lease to the packet. An empty /// logical slot must not hide reusable physical channels when the shared /// queue-ID namespace is full. + /// Also reports whether the owner's installed pair is ready to take the + /// command, so that the publication skips graph construction. fn acquire_render_with_ids( &self, packet: &Arc, mut next: Option, sleep: bool, - ) -> Result { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; + ) -> Result<(RenderLease, bool)> { + let pool = &self.admission; let started = super::super::now_ns(); - let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + let mut deadline_ns = + started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + let recovery_deadline_ns = started.saturating_add(recovery_wait_ns()); + // A capacity refusal samples the render domain and retries once; only + // a refusal after that sample waits, so no release can be missed. + let mut observed = None; + let mut capacity = WaitClock::default(); loop { self.admission_open()?; - if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } + if super::super::now_ns() >= deadline_ns { + if deadline_ns < recovery_deadline_ns && self.waiting_for_recovery()? { + deadline_ns = recovery_deadline_ns; + } else { + return Err(EBUSY); + } + } if packet.cancel_requested() { return Err(ECANCELED); } @@ -97,11 +104,13 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), false)? }; if reclaimed.is_none() { - self.wait_render_capacity(started, sleep)?; + self.wait_render_capacity(&mut observed, &mut capacity, sleep)?; } drop(reclaimed); continue; } + // The deadline check above decides whether a recovery extends the wait. + Err(error) if error == EBUSY => continue, Err(error) => return Err(error), } }; @@ -109,7 +118,10 @@ impl Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; match firmware.queues.ensure_render_ids(slot, self.owner) { - Ok(()) => return Ok(lease), + Ok(()) => { + let ready = firmware.queues.render_ready(slot, self.owner, &self.context)?; + return Ok((lease, ready)); + } Err(error) if error == ENOSPC => { let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; if reclaimed.is_some() { @@ -130,8 +142,9 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), true)?; let ready = firmware.queues.has_render_ids(self.owner); drop(state); - if reclaimed.is_none() && !ready { - self.wait_render_capacity(started, sleep)?; + // A full queue table makes admission reuse a free killed pair instead. + if reclaimed.is_none() && !ready && !pool.killed_pair_free() { + self.wait_render_capacity(&mut observed, &mut capacity, sleep)?; } drop(reclaimed); } @@ -140,31 +153,198 @@ impl Backend { } } - /// A pair is borrowed, still settling, or waiting for the release of the - /// scheduler it last ran for to be consumed and the pair parked. The event - /// worker observes both, so kick it on every poll; time out only after the - /// same bound as a capacity wait. - fn wait_render_busy(&self, since: &mut Option, sleep: bool) -> Result { - if !sleep { - return Err(FALLBACK); + /// Scheduler admission of a render packet. A work state of its context or + /// a pair admission it lacks becomes a fence that the return providing it + /// signals, so run_job finds both ready instead of waiting for them. + /// Errors, and anything else publication may wait for, are left to it. + pub(in crate::g17::runtime) fn admit_render(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + // Each check runs once more after sampling its domain, so the common + // case takes no domain lock and no return between check and wait is lost. + let work = packet.completion.work_state().ok()?; + loop { + match work.acquire(crate::g17::job::Engine::Render) { + Err(error) if error == EBUSY => { + let states = work.returns().observe(); + match work.acquire(crate::g17::job::Engine::Render) { + Err(error) if error == EBUSY => match work.returns().wait(states) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + }, + Err(_) => return None, + Ok(_) => {} + } + } + Err(_) => return None, + Ok(_) => {} + } + let request = Request::of(packet); + if self.admission.admits_render(self.owner, request) { + return None; + } + let observed = self.shared.renders.observe(); + if self.admission.admits_render(self.owner, request) { + return None; + } + self.shared.queue_events(); + match self.shared.renders.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + + /// Conditions which end a publication that has not become visible yet. + fn publication_open(&self, packet: &Packet) -> Result { + self.admission_open()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); } - let now = super::super::now_ns(); - if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { + Ok(()) + } + + /// Blocks on `domain` until a notification after `observed`, or the + /// deadline, which only bounds the wait. Slices of at most 10 ms let the + /// publisher observe cancellation, which has no notification here. + fn wait_domain( + &self, + domain: &crate::g17::availability::Availability, + observed: u64, + deadline_ns: u64, + packet: &Packet, + ) -> Result { + self.publication_open(packet)?; + let remaining = deadline_ns.saturating_sub(super::super::now_ns()); + if remaining == 0 { return Err(ETIMEDOUT); } + domain.block(observed, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// A recovery is pending, or this owner's pair is spared and draining after one: an + /// innocent publication waits up to RECOVERY_WAIT_MS for it instead of failing. + fn waiting_for_recovery(&self) -> Result { + let state = self.shared.state.lock(); + let firmware = (*state).as_deref().ok_or(ENODEV)?; + Ok(firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + || firmware.queues.admission.spared_quarantined(self.owner)) + } + + /// The deadline of the wait `clock` measures (WaitClock::deadline). + fn wait_deadline(&self, clock: &mut WaitClock, bound_ns: u64) -> Result { + let recovering = self.waiting_for_recovery()?; + Ok(clock.deadline(super::super::now_ns(), recovering, bound_ns)) + } + + /// A pair is borrowed, still settling, waiting for firmware room or for the + /// release of the scheduler it last ran for. Every pair return, retirement + /// and teardown step signals the render domain, and each event pass does + /// while a publisher waits; the deadline is only the failure bound. + fn wait_render_busy( + &self, + clock: &mut WaitClock, + observed: &mut Option, + sleep: bool, + packet: &Packet, + ) -> Result { + if !sleep { + return Err(FALLBACK); + } + let deadline = self.wait_deadline(clock, super::super::WORK_RING_WAIT_NS)?; + // Sample, then retry once: only a refusal after the sample waits. + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; self.shared.queue_events(); - fsleep(Delta::from_micros(200)); - Ok(()) + self.wait_domain(&self.shared.renders, sample, deadline, packet) } - /// A closing owner may still hold its pair until exact retirement. Wait - /// outside both the device mutex and preparation gate so the event worker - /// and recovery can return capacity. No packet is firmware-visible here. - fn wait_render_capacity(&self, started: u64, sleep: bool) -> Result { + /// Render capacity is short: return what idle owners hold, then block until + /// a pair, lease or owner returns. No packet is firmware-visible here, and + /// neither the device mutex nor a preparation lease is held while waiting. + fn wait_render_capacity( + &self, + observed: &mut Option, + clock: &mut WaitClock, + sleep: bool, + ) -> Result { if !sleep { return Err(FALLBACK); } - self.wait_capacity(started) + if self.reclaim_render_capacity()? { + *observed = None; + return Ok(()); + } + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; + self.shared.queue_events(); + self.admission_open()?; + let remaining = self + .wait_deadline(clock, super::super::WORK_RING_WAIT_NS)? + .saturating_sub(super::super::now_ns()); + if remaining == 0 { + return Err(EBUSY); + } + self.shared + .renders + .block(sample, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// Gives idle render backing back under capacity pressure. Returns whether + /// anything was returned, in which case the caller retries at once. + fn reclaim_render_capacity(&self) -> Result { + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Ok(false); + } + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + let mut published = 0; + if !firmware.render_control_backpressured() { + (published, _) = firmware.queues.vacate_idle_pools( + &firmware.init, + 0, + super::super::now_ns(), + 0, + )?; + if published != 0 { + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + } + Ok(published != 0 + || !deferred.render.is_empty() + || !deferred.vacated.is_empty() + || !deferred.bindings.is_empty()) + })(); + deferred.finish(); + match result { + Err(error) if error == EAGAIN => Ok(false), + result => result, + } } fn render_with_lease( @@ -185,14 +365,45 @@ impl Backend { // A publication from the submitting ioctl (an early lease) never // sleeps for contention; it falls back to the scheduler instead. let sleep = early.is_none(); - let admission = self.acquire_render_with_ids(packet, early, sleep)?; + let mut early = early; + // The pair this owner last published to directly: a lease on it goes + // straight to the direct path, whose lend checks everything the + // identity check would. + let known = self.direct_slot.load(Ordering::Relaxed); + let lease = match early.take() { + Some(lease) => Some(lease), + None if known != 0 => self + .admission + .try_acquire_render(self.owner, Request::of(packet)) + .ok() + .flatten(), + None => None, + }; + let early = match lease { + Some(lease) if u32::from(lease.slot()) == known => { + let slot = lease.slot(); + packet.completion.attach_render_lease(lease)?; + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + packet.completion.detach_render_lease() + } + lease => lease, + }; + let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; - let mut state_polls = 0; - let mut state_delay = 20i64; - let mut busy_since = None; - let mut recovery_polls = 0; - let capacity_started = super::super::now_ns(); + if ready { + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + } + let mut busy = WaitClock::default(); + let mut capacity = WaitClock::default(); + let mut state_clock = WaitClock::default(); + let mut recovery_since = None; + let mut seen = None; + let mut states = None; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -210,7 +421,7 @@ impl Backend { } else { self.enter_preparation(false)? }; - let result = packet.completion.work_state().and_then(|state| state.acquire()); + let result = packet.completion.work_state().and_then(|state| state.acquire(crate::g17::job::Engine::Render)); if preparation.is_current() { result } else { @@ -223,54 +434,72 @@ impl Backend { if !sleep { return Err(FALLBACK); } - if state_polls == 1000 { - return Err(ETIMEDOUT); + // Every node of this context belongs to unretired work: + // sample, retry once, then wait for the next returned one. + let work = packet.completion.work_state()?; + match states.take() { + None => states = Some(work.returns().observe()), + // Work states free as earlier, possibly spared, work retires. + Some(sample) => { + let deadline = self + .wait_deadline(&mut state_clock, super::super::WORK_RING_WAIT_NS)?; + self.wait_domain(work.returns(), sample, deadline, packet)? + } } - state_polls += 1; - fsleep(Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); continue; } result => { result?; } } - match self.ensure_render(packet, slot, sleep) { + // A pair found ready under the identity check needs no build; any + // retry revalidates through ensure_render. + let ensured = if core::mem::take(&mut ready) { + Ok(()) + } else { + self.ensure_render(packet, slot, sleep) + }; + match ensured { Err(error) if error == ENOSPC => { - self.wait_render_capacity(capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; continue; } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy_since, sleep)?; + self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; continue; } result => result?, } match self.prepare_render(slot, packet, deferred, sleep) { Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_render_capacity(capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, &mut capacity, sleep)?; continue; } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { - let recovering = { + let (recovering, gate) = { let state = self.shared.state.lock(); let firmware = (*state).as_deref().ok_or(ENODEV)?; - firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + ( + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0, + self.shared.preparations.snapshot(), + ) }; if recovering { if !sleep { return Err(FALLBACK); } - if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { - return Err(ETIMEDOUT); - } - recovery_polls += 1; + // Recovery closes the preparation gate before touching + // any queue; the next pass blocks in enter() until it + // reopens. The worker raises the closure. + let since = *recovery_since.get_or_insert(super::super::now_ns()); self.shared.queue_events(); - fsleep(Delta::from_millis(1)); + self.shared + .preparations + .wait_closure(gate, since.saturating_add(recovery_wait_ns()))?; } else if error == EBUSY { - self.wait_render_busy(&mut busy_since, sleep)?; + self.wait_render_busy(&mut busy, &mut seen, sleep, packet)?; } else if !sleep { return Err(FALLBACK); } @@ -284,19 +513,9 @@ impl Backend { if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { return false; } - let admission = { - let state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref() else { - return false; - }; - match firmware - .queues - .admission - .try_acquire_render(self.owner, Request::of(packet)) - { - Ok(Some(lease)) => lease, - _ => return false, - } + let admission = match self.admission.try_acquire_render(self.owner, Request::of(packet)) { + Ok(Some(lease)) => lease, + _ => return false, }; if !packet.begin_publication() { packet.completion.fail_unpublished(ECANCELED); @@ -448,15 +667,28 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } + let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; + // Every render publication, with or without the device mutex, holds the + // port's render lock: the render work rings have one host producer at a + // time and attachment overlaps are ordered by publication order. + let port = self.port.clone(); + let _render = port.render_lock(); // Both outer destinations must have room before pool accounting or - // either inner queue becomes visible. All host producers share this lock. + // either inner queue becomes visible. let priority = packet.context.policy().priority(); if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } - let dependencies = self.queues.render_dependencies(owner, packet, state)?; + if let Some(footprint) = packet.footprint.as_ref() { + dependencies.hazard_count = self.queues.admission.publication_barriers( + owner, + slot, + footprint, + &mut dependencies.hazards, + )?; + } let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; // An owner change taken from a live owner which closed meanwhile: its // first command waits until that scheduler's release has been consumed. @@ -505,3 +737,57 @@ impl crate::g17::Firmware { result } } + +fn recovery_wait_ns() -> u64 { + u64::from(crate::g17::timeout::RECOVERY_WAIT_MS) * 1_000_000 +} + +/// One bounded wait (Backend::wait_deadline). +#[derive(Default)] +struct WaitClock { + first: Option, + calm: u64, +} + +impl WaitClock { + /// `bound_ns` after the later of the first call and the last call that saw a recovery + /// pending or this owner's spared pair draining (so the usual bound applies afresh once a + /// recovery ends), and at most RECOVERY_WAIT_MS after the first call. + fn deadline(&mut self, now: u64, recovering: bool, bound_ns: u64) -> u64 { + let first = *self.first.get_or_insert(now); + self.calm = if recovering { now } else { self.calm.max(first) }; + first + .saturating_add(recovery_wait_ns()) + .min(self.calm.saturating_add(bound_ns)) + } +} + +#[kunit_tests(asahi_g17_wait_clock)] +mod tests { + use super::*; + + const BOUND: u64 = 2_000_000_000; + + #[test] + fn a_calm_wait_keeps_its_first_deadline() { + let mut clock = WaitClock::default(); + assert_eq!(clock.deadline(100, false, BOUND), 100 + BOUND); + assert_eq!(clock.deadline(500, false, BOUND), 100 + BOUND); + } + + #[test] + fn the_bound_starts_afresh_once_a_recovery_ends() { + let mut clock = WaitClock::default(); + clock.deadline(100, false, BOUND); + assert_eq!(clock.deadline(1_000, true, BOUND), 1_000 + BOUND); + assert_eq!(clock.deadline(2_000, false, BOUND), 1_000 + BOUND); + } + + #[test] + fn a_recovery_cannot_extend_past_the_recovery_wait() { + let mut clock = WaitClock::default(); + clock.deadline(100, true, BOUND); + let limit = 100 + recovery_wait_ns(); + assert_eq!(clock.deadline(limit - 1, true, BOUND), limit); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index eb04f80361955..fffa6d28dd289 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -38,7 +38,7 @@ impl RenderState { return Ok(false); } if let Some(pair) = entry.pair.as_ref() { - if !pair.idle_for_context(context)? { + if !pair.idle_for_context(context)? && !pair.executed_for_kill(context) { return Ok(false); } } @@ -46,6 +46,14 @@ impl RenderState { Ok(true) } + /// Some render pair runs another context's commands. + pub(in crate::g17::runtime) fn foreign_in_flight(&self, context: &Arc) -> bool { + self.entries.iter().flatten().any(|entry| { + !entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) + && entry.pair.as_ref().is_some_and(|pair| pair.in_flight()) + }) + } + pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { context.render_pools_released(|id, control| self.pools.released(id, control)) && self @@ -115,6 +123,7 @@ impl RenderState { if !out.has_room(count, 1) { return Err(EBUSY); } + let linked = context.firmware_may_link(); for entry in self.entries.iter_mut().flatten() { if !entry .context @@ -132,7 +141,14 @@ impl RenderState { let (mut parked, retired) = KBox::into_inner(pair).park(); // This park follows the consumed release of `context`. parked.release_consumed(context.scheduler_va()); - *target = Some(retired); + if linked { + // Guard (kill before free): the firmware may still walk this context's work + // list into the graph, so it is not freed or recycled before the firmware stops. + kernel::pr_warn_once!("G17: keeping render graphs of an unkilled context that still links work\n"); + keep_graph(&mut self.kept_graphs, retired); + } else { + *target = Some(retired); + } entry.parked = Some(parked); // Detached backing retains these references until the worker runs. entry.context = None; @@ -147,8 +163,12 @@ impl RenderState { if let Some(id) = self.retired[index].buffer_id() { buffers.unreserve_owner(id)?; } - let target = out.binding_slot()?; - *target = Some(self.retired.remove(index).map_err(|_| EIO)?); + let retired = self.retired.remove(index).map_err(|_| EIO)?; + if linked { + keep_graph(&mut self.kept_graphs, retired); + } else { + *out.binding_slot()? = Some(retired); + } } Ok(()) } @@ -173,3 +193,23 @@ impl Registry { Ok(()) } } + +/// Graphs that could not be recorded in `kept_graphs` and were leaked instead, for debugfs. +pub(super) static LEAKED_GRAPHS: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); + +/// Keeps `graph` until the firmware stops; if even that cannot be recorded, it is leaked +/// rather than freed. +fn keep_graph( + kept: &mut KVec, + graph: crate::g17::queue::render::RetiredBinding, +) { + if kept.reserve(1, GFP_KERNEL).is_ok() { + if let Err(error) = kept.push_within_capacity(graph) { + LEAKED_GRAPHS.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + core::mem::forget(error.0); + } + } else { + LEAKED_GRAPHS.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + core::mem::forget(graph); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index f8020e8962f84..0222ae4c478c8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -100,6 +100,8 @@ impl ReclaimBatch { struct Service<'a> { render: &'a mut super::render::RenderState, + admission: &'a crate::g17::queue::admission::Pool, + objects: &'a Arc, compute: &'a [Option], buffers: &'a mut BufferIds, init: &'a InitData, @@ -115,7 +117,7 @@ impl Host for Service<'_> { self.compute .iter() .flatten() - .filter_map(|entry| entry.queue.as_ref()) + .filter_map(|entry| entry.queue.as_deref()) .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { @@ -137,8 +139,15 @@ impl Host for Service<'_> { self.primary.notify(MSG_CONTROL_NOTIFY) } fn detach(&mut self, context: &Arc) -> Result { + let linked = context.retain_linked_work(self.objects); + let (_, _, slots) = self.render.exit_scope(context); + crate::g17::trace::teardown(5, u64::from(context.id()), slots, u64::from(linked)); self.render .detach_context(context, self.buffers, self.reclaim)?; + // The firmware consumed this context's scheduler release after its kill (if any) + // was acknowledged and its pairs went idle: it no longer names these pairs, so + // their kill hold ends here rather than at the next processor stop. + self.admission.release_killed_slots(slots); // detach_context preflights this slot together with every graph slot. // Retain it before Pending removes its reference under the device lock. self.reclaim.context(context) @@ -173,6 +182,7 @@ impl Firmware { self.queues.render.observe_teardown_releases(&self.init)?; let recovery = self.recovery.pending() || self.init.recovery_state()? != 0; let Registry { + admission, render, compute, buffers, @@ -181,6 +191,8 @@ impl Firmware { } = &mut self.queues; let mut service = Service { render, + admission, + objects: &self.pool, compute, buffers, init: &self.init, @@ -190,10 +202,17 @@ impl Firmware { teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); + let mut consumed = false; while teardown .take_consumed(consumer, producer, &mut service)? .is_some() - {} + { + consumed = true; + } + if consumed { + // A channel whose last scheduler release is consumed can change owner. + self.primary.state.shared.channels.notify(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 53518a76e91a4..fa2a72a77f060 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -24,6 +24,45 @@ struct Entry { phase: Phase, armed: u64, announce_pending: bool, + render_kill: Option, + /// Since when, and whether reported, a render-only kill has waited for other contexts' + /// compute to pause. + kill_wait: Option<(u64, bool)>, + /// A render-only kill of this context was refused (counted once per context). + kill_refused: bool, +} + +/// The kill of a closed context whose compute side is clean, published so that its render +/// pairs' executed but unstamped commands can be released (see `Pair::firmware_drained`). +/// The scheduler page's killed state, not this record, proves the kill to the pairs. +#[derive(Clone, Copy)] +struct RenderKill { + cookie: u64, + published: u64, + announced: bool, + acknowledged: bool, +} + +impl RenderKill { + fn new(cookie: u64, now: u64) -> Self { + Self { + cookie, + published: now, + announced: false, + acknowledged: false, + } + } + /// Only this kill's exact cookie acknowledges it, once. + fn acknowledge(&mut self, cookie: u64) -> bool { + if cookie == 0 || cookie != self.cookie || self.acknowledged { + return false; + } + self.acknowledged = true; + true + } + fn awaiting(&self, now: u64, window: u64) -> bool { + !self.acknowledged && now.saturating_sub(self.published) < window + } } /// Publication and announcement errors retain the entry; no timeout frees @@ -104,6 +143,9 @@ impl Pending { phase, armed: now, announce_pending: false, + render_kill: None, + kill_wait: None, + kill_refused: false, }; self.entries.push(entry, GFP_KERNEL)?; Ok(()) @@ -233,6 +275,94 @@ impl Pending { Ok(None) } + /// A render-only kill of the closed context at `index` was published. + pub(crate) fn render_kill_published(&self, index: usize) -> bool { + self.entries.get(index).is_some_and(|entry| entry.render_kill.is_some()) + } + + /// Record a published render-only kill. Publication is irrevocable: the record stays + /// with the entry until the context's scheduler release is consumed. + pub(crate) fn record_render_kill(&mut self, index: usize, cookie: u64, now: u64) -> Result { + let entry = self.entries.get_mut(index).ok_or(EINVAL)?; + if entry.render_kill.is_some() { + return Err(EINVAL); + } + entry.render_kill = Some(RenderKill::new(cookie, now)); + Ok(()) + } + + /// Ring the doorbell for published render-only kills not yet announced. A failed + /// doorbell is retried on a later pass and never duplicates the record. + pub(crate) fn announce_render_kills(&mut self, mut notify: impl FnMut() -> Result) { + for entry in self.entries.iter_mut() { + if let Some(kill) = entry.render_kill.as_mut() { + if !kill.announced && !kill.acknowledged && notify().is_ok() { + kill.announced = true; + } + } + } + } + + /// Only the exact cookie of an unacknowledged render-only kill is accepted. + pub(crate) fn acknowledge_render_kill(&mut self, cookie: u64) -> bool { + self.entries + .iter_mut() + .filter_map(|entry| entry.render_kill.as_mut()) + .any(|kill| kill.acknowledge(cookie)) + } + + /// A render-only kill awaits its acknowledgement within the polling window. + pub(crate) fn render_kill_polling(&self, now: u64, window: u64) -> bool { + self.entries + .iter() + .any(|entry| entry.render_kill.is_some_and(|kill| kill.awaiting(now, window))) + } + + /// The render-only kill of the entry at `index` waits; how long, and whether this is the + /// first time it crossed `slow`. + pub(crate) fn note_kill_wait(&mut self, index: usize, now: u64, slow: u64) -> (u64, bool) { + let Some(entry) = self.entries.get_mut(index) else { + return (0, false); + }; + let (since, warned) = entry.kill_wait.get_or_insert((now, false)); + let waited = now.saturating_sub(*since); + let crossed = waited >= slow && !*warned; + *warned |= crossed; + (waited, crossed) + } + + /// The render-only kill was published: how long it waited, if it did. + pub(crate) fn take_kill_wait(&mut self, index: usize, now: u64) -> u64 { + self.entries + .get_mut(index) + .and_then(|entry| entry.kill_wait.take()) + .map_or(0, |(since, _)| now.saturating_sub(since)) + } + + /// A render-only kill of the context at `index` was refused: it no longer waits, and this + /// returns whether it is the first refusal for this context. + pub(crate) fn refuse_render_kill(&mut self, index: usize) -> bool { + let Some(entry) = self.entries.get_mut(index) else { + return false; + }; + entry.kill_wait = None; + !core::mem::replace(&mut entry.kill_refused, true) + } + + /// The context at `index` is not waiting for a render-only kill (any more). + pub(crate) fn clear_kill_wait(&mut self, index: usize) { + if let Some(entry) = self.entries.get_mut(index) { + entry.kill_wait = None; + } + } + + /// How long each waiting render-only kill has waited. + pub(crate) fn kill_waits(&self, now: u64) -> impl Iterator + '_ { + self.entries + .iter() + .filter_map(move |entry| entry.kill_wait.map(|(since, _)| now.saturating_sub(since))) + } + /// Poll for a bounded window at close and again at actual release publication. /// Failed announcements remain retryable on later service even after polling /// expires. Time passage never substitutes for a firmware-consumption witness. @@ -242,3 +372,21 @@ impl Pending { }) } } + +#[kernel::macros::kunit_tests(asahi_g17_render_kill)] +mod render_kill_tests { + use super::RenderKill; + + #[test] + fn only_the_exact_cookie_acknowledges_a_render_kill_once() { + let mut kill = RenderKill::new(7, 1_000); + assert!(kill.awaiting(1_000, 10)); + assert!(!kill.acknowledge(0)); + assert!(!kill.acknowledge(8)); + assert!(kill.awaiting(1_009, 10)); + assert!(!kill.awaiting(1_010, 10)); + assert!(kill.acknowledge(7)); + assert!(!kill.acknowledge(7)); + assert!(!kill.awaiting(1_001, 10)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index d951bd164d6e9..ce003e6b18196 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -18,6 +18,13 @@ pub(crate) fn transaction_grace(renewals: u8) -> bool { renewals < MAX_RENEWALS pub(crate) const POLL_MS: u32 = 2000; const EXECUTION_RENEWALS: u8 = 4; +/// How long an innocent client's publication waits for a recovery to settle and its +/// spared work to drain, instead of failing with EBUSY: the watchdog defers work behind +/// another VM at most this long before it settles it (`classify`; ray work gets the +/// execution renewals on top). +pub(crate) const RECOVERY_WAIT_MS: u32 = + POLL_MS * (1 + EXECUTION_RENEWALS as u32 + MAX_RENEWALS as u32); + /// A background control/replay poll has no scheduler fence to expire it. /// Its timestamp belongs to the exact transaction and is never renewed by work. pub(crate) fn background_stalled(since: Option, now: u64) -> bool { @@ -53,6 +60,37 @@ pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) true } +/// Polls a published job may be renewed while the GPU keeps retiring work. A job +/// shares the GPU's cores with every other runnable queue, so its wall time grows +/// with the other clients' work: beside four other clients a 1.5 s dispatch took up +/// to 5 s, and lower-priority work waits behind higher-priority work. It has not +/// hung while the GPU keeps completing commands. Fourteen renewals bound the wait at +/// thirty seconds, after which a job that still has not finished fails as before. +const SHARED_RENEWALS: u8 = 14; + +/// The GPU counts as live while some command retired this recently. Under heavy +/// sharing the longest gap between retirements measured 1.2 s; a GPU that retires +/// nothing for this long has stopped. +pub(crate) const LIVENESS_NS: u64 = 5_000_000_000; + +/// Whether some command retired within [`LIVENESS_NS`] (`since_retired`). +pub(crate) fn retiring(since_retired: Option) -> bool { + since_retired.is_some_and(|since| since < LIVENESS_NS) +} + +/// Renews a published job's deadline while the GPU is `shared`, at most +/// [`SHARED_RENEWALS`] times. It is shared while some command retires +/// ([`retiring`]), or while a long job of another queue runs beside a long job: +/// two ray traversal dispatches of about 6 s each took 10 to 17 s side by side, and +/// neither retires anything until it ends. +pub(crate) fn renew_shared(published: bool, shared: bool, renewals: &mut u8) -> bool { + if !published || !shared || *renewals >= SHARED_RENEWALS { + return false; + } + *renewals += 1; + true +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. @@ -215,6 +253,37 @@ mod tests { assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); } #[test] + fn a_live_gpu_renews_a_published_job_for_thirty_seconds() { + let mut renewals = 0; + for poll in 1..=14u8 { + assert!(renew_shared(true, retiring(Some(4_900_000_000)), &mut renewals)); + assert_eq!(renewals, poll); + } + assert!(!renew_shared(true, true, &mut renewals)); + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 30_000); + } + #[test] + fn a_stopped_gpu_or_unpublished_job_keeps_the_deadline() { + let mut renewals = 0; + assert!(!renew_shared(true, retiring(Some(LIVENESS_NS)), &mut renewals)); + assert!(!renew_shared(true, retiring(None), &mut renewals)); + assert!(!renew_shared(false, true, &mut renewals)); + assert_eq!(renewals, 0); + } + #[test] + fn a_long_job_beside_another_shares_the_gpu_without_retirements() { + // Two traversal dispatches side by side: nothing retires, yet both progress. + let mut renewals = 0; + let long_job_beside = true; + let shared = retiring(Some(LIVENESS_NS + 1)) || long_job_beside; + for poll in 1..=14u8 { + assert!(renew_shared(true, shared, &mut renewals)); + assert_eq!(renewals, poll); + } + // The thirty-second bound still ends a pair that never finishes. + assert!(!renew_shared(true, true, &mut renewals)); + } + #[test] fn exhausted_or_invalid_credit_never_wraps() { for value in 4..=u8::MAX { let mut renewals = value; diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index b2a6cf6e8bfe3..b0f2861382ad7 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -8,19 +8,41 @@ use core::{cell::RefCell, ops::Range}; use kernel::prelude::*; const MAX_RANGES: usize = 48; +// The persistent proof must retain a deep queue's rotating command buffers. +// Keep the transient parser request set small to bound its stack frame. +const MAX_CACHED_RANGES: usize = 128; type Request = (u64, u64, u8); +/// Match the page footprint already checked by `Vm::covers_range_locked`. +fn pages((address, size, access): Request) -> Option { + if size == 0 || access == 0 || access & !3 != 0 { + return None; + } + let mask = mmu::UAT_PGSZ as u64 - 1; + let end = address.checked_add(size)?; + let end = end.checked_add(mask)? & !mask; + let start = address & !mask; + Some((start, end.checked_sub(start)?, access)) +} + +fn contains((start, size, access): Request, (other, length, needed): Request) -> bool { + access & needed == needed + && other >= start + && other.checked_add(length).is_some_and(|end| + start.checked_add(size).is_some_and(|limit| end <= limit)) +} + #[derive(Clone, Copy)] -struct Requests { - ranges: [Request; MAX_RANGES], +struct Requests { + ranges: [Request; CAPACITY], count: usize, overflow: bool, } -impl Requests { +impl Requests { const fn new() -> Self { Self { - ranges: [(0, 0, 0); MAX_RANGES], + ranges: [(0, 0, 0); CAPACITY], count: 0, overflow: false, } @@ -30,24 +52,55 @@ impl Requests { &self.ranges[..self.count] } + fn covers(&self, request: Request) -> bool { + self.as_slice().iter().any(|&range| contains(range, request)) + } + fn insert(&mut self, request: Request) { - if self.as_slice().contains(&request) { + let Some(mut request) = pages(request) else { + self.overflow = true; + return; + }; + if self.covers(request) { return; } - if self.count == MAX_RANGES { + // Merge only equal permissions, so an adjacent writable page never + // grants write access to a read-only page. Restart after each merge + // because it can join a range visited earlier. + let mut index = 0; + while index < self.count { + let range = self.ranges[index]; + let end = request.0 + request.1; + let other_end = range.0 + range.1; + if contains(request, range) + || (request.2 == range.2 && request.0 <= other_end && range.0 <= end) + { + if request.2 == range.2 { + let start = request.0.min(range.0); + request = (start, end.max(other_end) - start, request.2); + } + self.count -= 1; + self.ranges[index] = self.ranges[self.count]; + index = 0; + } else { + index += 1; + } + } + if self.count == CAPACITY { self.overflow = true; return; } self.ranges[self.count] = request; self.count += 1; } + } -/// One logical queue's bounded cache of exact address, size and permission tuples. +/// One logical queue's bounded cache of validated page ranges and permissions. /// Mapping mutations make the epoch unavailable until their final page-table write. pub(crate) struct Proof { epoch: Option, - requests: Requests, + requests: Requests, } impl Proof { @@ -64,7 +117,7 @@ impl Proof { && requests .as_slice() .iter() - .all(|request| self.requests.as_slice().contains(request)) + .all(|&request| self.requests.covers(request)) } fn record(&mut self, epoch: Option, requests: Requests) { @@ -79,7 +132,10 @@ impl Proof { } // An incomplete union cannot be reused; retain the latest complete set. self.epoch = if requests.overflow { None } else { epoch }; - self.requests = requests; + self.requests = Requests::new(); + for &request in requests.as_slice() { + self.requests.insert(request); + } } /// The VM-job guard must already pin the mappings for this command. Keep the @@ -144,3 +200,101 @@ impl AddressSpace for Space<'_> { } } } + + +#[kunit_tests(asahi_g17_validation_pages)] +mod tests { + use super::*; + const PAGE: u64 = mmu::UAT_PGSZ as u64; + + fn requests(range: Request) -> Requests { + let mut result = Requests::new(); + result.insert(range); + result + } + + #[test] + fn different_offsets_share_the_already_validated_page() { + let mut proof = Proof::new(); + proof.record(Some(7), requests((PAGE + 4, 4, 3))); + assert!(proof.covers(Some(7), &requests((PAGE + 0x100, 8, 1)))); + assert!(proof.covers(Some(7), &requests((2 * PAGE - 8, 8, 2)))); + assert!(!proof.covers(Some(7), &requests((2 * PAGE - 8, 16, 1)))); + assert!(!proof.covers(Some(8), &requests((PAGE + 4, 4, 1)))); + assert!(!proof.covers(None, &requests((PAGE + 4, 4, 1)))); + } + + #[test] + fn adjacent_equal_permissions_merge_but_holes_do_not() { + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((3 * PAGE, 1, 1)); + assert_eq!(ranges.count, 2); + assert!(!ranges.covers((2 * PAGE, PAGE, 1))); + ranges.insert((2 * PAGE, 1, 1)); + assert_eq!(ranges.count, 1); + assert_eq!(ranges.as_slice(), &[(PAGE, 3 * PAGE, 1)]); + } + + #[test] + fn adjacent_permissions_never_grant_access_to_another_page() { + let mut proof = Proof::new(); + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((2 * PAGE, 1, 3)); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((PAGE, 1, 2)))); + assert!(proof.covers(Some(1), &requests((2 * PAGE, 1, 2)))); + assert!(!proof.covers(Some(1), &requests((PAGE, 2 * PAGE, 3)))); + } + + #[test] + fn stronger_coverage_removes_only_fully_covered_ranges() { + let mut ranges = requests((PAGE, PAGE, 1)); + ranges.insert((2 * PAGE, PAGE, 2)); + ranges.insert((PAGE, 2 * PAGE, 3)); + assert_eq!(ranges.as_slice(), &[(PAGE, 2 * PAGE, 3)]); + assert!(!ranges.covers((0, PAGE, 1))); + } + + #[test] + fn persistent_cache_retains_a_deep_rotating_working_set() { + let mut proof = Proof::new(); + for index in 0..64 { + proof.record(Some(7), requests((index * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 64); + for index in 0..64 { + assert!(proof.covers(Some(7), &requests((index * 2 * PAGE + 16, 8, 1)))); + } + assert_eq!(core::mem::size_of::(), core::mem::size_of::>()); + } + + #[test] + fn persistent_overflow_keeps_only_a_new_complete_proof() { + let mut proof = Proof::new(); + for index in 0..=MAX_CACHED_RANGES { + proof.record(Some(7), requests((index as u64 * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 1); + assert!(!proof.covers(Some(7), &requests((0, 8, 1)))); + assert!(proof.covers(Some(7), + &requests((MAX_CACHED_RANGES as u64 * 2 * PAGE, 8, 1)))); + } + + #[test] + fn invalid_rounding_and_full_cache_disable_reuse() { + for range in [(0, 0, 1), (u64::MAX - 1, 1, 1), (0, 1, 0), (0, 1, 4)] { + assert!(pages(range).is_none()); + let mut proof = Proof::new(); + proof.record(Some(1), requests(range)); + assert!(!proof.covers(Some(1), &requests((0, 1, 1)))); + } + let mut ranges = Requests::new(); + for index in 0..=MAX_RANGES { + ranges.insert(((2 * index) as u64 * PAGE, PAGE, 1)); + } + assert!(ranges.overflow); + let mut proof = Proof::new(); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((0, PAGE, 1)))); + } +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 72238159d0475..f100820a0c75e 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -41,9 +41,7 @@ use crate::{ AsahiDrmDevice, AsahiDrmDriver, // }, - file, - memacct, - mmu, + file, memacct, mmu, }; const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; @@ -133,9 +131,6 @@ pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { Ok(()) } -unsafe impl Send for AsahiObject {} -unsafe impl Sync for AsahiObject {} - /// A shared reference to a GEM object for this driver. pub(crate) struct ObjectRef { /// The underlying GEM object reference @@ -410,6 +405,23 @@ impl AsahiObject { } } +/// Populates and pins a new user object through its DMA mapping, before any CPU view exists. +/// +/// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the +/// freshly zeroed pages to the point of coherency, so that no dirty line can later overwrite data +/// written through a write-combined view. The pin keeps reclaim from creating a new cacheable +/// alias. Only used when the GPU asks for it ([`crate::gpu::Gpu::prepare_new_objects`]). +/// +pub(crate) fn prepare_new_object( + gem: &Object, + dev: &kernel::device::Device, +) -> Result { + // The native backend requires a non-coherent DMA device. Use its DMA mapping + // rather than interpreting DMA addresses as CPU physical addresses. + let _sgt = held_sg_table(gem, dev)?; + Ok(()) +} + /// Purge advice for a VM-private, non-exportable object. Returns whether the /// contents are retained (0 once purged). pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { @@ -419,6 +431,26 @@ pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { Ok(u32::from(gem.madvise(i32::from(dontneed)))) } +/// Large user BOs have a cheap ENOMEM fallback instead of per-folio reclaim. +/// Keep ordinary resource allocations and kernel-owned backing on their existing +/// policy. 256 MiB also matches Mesa's maximum reusable BO-cache footprint. +pub(crate) fn user_pages_no_reclaim(size: usize, g17: bool) -> bool { + g17 && size >= 256 * 1024 * 1024 +} + +#[kernel::macros::kunit_tests(asahi_gem_allocation_policy)] +mod allocation_policy_tests { + use super::*; + + #[test] + fn large_user_backing() { + assert!(!user_pages_no_reclaim(256 * 1024 * 1024 - 1, true)); + assert!(user_pages_no_reclaim(256 * 1024 * 1024, true)); + assert!(user_pages_no_reclaim(3_257_113_344, true)); + assert!(!user_pages_no_reclaim(3_257_113_344, false)); + } +} + /// Populates and pins a new user object through its DMA mapping, before any CPU view exists. /// /// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the @@ -429,7 +461,7 @@ pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { /// The DMA mapping only cleans if the device is not marked DMA-coherent (G15's synthesized node /// has no `dma-coherent`). Should it ever be marked so, the pages are cleaned here instead. #[cfg(CONFIG_DRM_ASAHI_G15)] -pub(crate) fn prepare_new_object(gem: &Object) -> Result { +pub(crate) fn prepare_new_object_g15(gem: &Object) -> Result { // SAFETY: G15 objects belong to the retained bound platform device; user object // creation occurs only after registration and before a file can outlive its device. let dev: &kernel::device::Device = gem.dev().as_ref().as_ref(); @@ -466,28 +498,7 @@ pub(crate) fn prepare_new_object(gem: &Object) -> Result { Ok(()) } - #[cfg(CONFIG_DRM_ASAHI_G15)] -extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } - -#[cfg(not(CONFIG_DRM_ASAHI_G15))] -pub(crate) fn prepare_new_object(_gem: &Object) -> Result { Err(EINVAL) } -/// Large user BOs have a cheap ENOMEM fallback instead of per-folio reclaim. -/// Keep ordinary resource allocations and kernel-owned backing on their existing -/// policy. 256 MiB also matches Mesa's maximum reusable BO-cache footprint. -pub(crate) fn user_pages_no_reclaim(size: usize, g17: bool) -> bool { - g17 && size >= 256 * 1024 * 1024 -} - -#[kernel::macros::kunit_tests(asahi_gem_allocation_policy)] -mod allocation_policy_tests { - use super::*; - - #[test] - fn large_user_backing() { - assert!(!user_pages_no_reclaim(256 * 1024 * 1024 - 1, true)); - assert!(user_pages_no_reclaim(256 * 1024 * 1024, true)); - assert!(user_pages_no_reclaim(3_257_113_344, true)); - assert!(!user_pages_no_reclaim(3_257_113_344, false)); - } +extern "C" { + fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index be65af433a633..3da16c10696e3 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -35,6 +35,7 @@ use kernel::{ }, macros::versions, new_mutex, + new_spinlock, prelude::*, soc::apple::rtkit, sync::{ @@ -44,6 +45,7 @@ use kernel::{ }, Arc, Mutex, + SpinLock, UniqueArc, // }, time::{ @@ -84,6 +86,7 @@ use crate::{ queue, regs, stats, + stats_raw, workqueue, // }; @@ -258,11 +261,13 @@ pub(crate) struct GpuManager { pipes: PipeChannels::ver, event_manager: Arc, stats: Arc, + /// Last raw firmware statistics of this device, exported read-only + /// through `stats_raw`. + raw_stats: Arc>, buffer_mgr: buffer::BufferManager::ver, ids: SequenceIDs, - #[allow(clippy::vec_box)] #[pin] - garbage_contexts: Mutex>>>, + garbage_contexts: Mutex>, } /// Operations the DRM driver performs on a GPU, independent of its firmware interface. @@ -275,6 +280,11 @@ pub(crate) trait Gpu: Send + Sync { /// The firmware statistics snapshot of this GPU, if its firmware reports any. fn stats_snapshot(&self) -> Option>; + /// The raw firmware statistics snapshot of this GPU, if its firmware reports any. + fn raw_stats_snapshot(&self) -> Option>> { + None + } + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -291,7 +301,7 @@ pub(crate) trait Gpu: Send + Sync { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; /// Whether unmapped GPU reads return zero (fault_control programmed into the firmware). fn supports_soft_faults(&self) -> bool { - *crate::module_parameters::fault_control.value() == 0xb + crate::module_parameters::fault_control.value() == 0xb } /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { @@ -305,14 +315,14 @@ pub(crate) trait Gpu: Send + Sync { fn feature_flags(&self) -> u64 { let mut flags = 0; if self.supports_vm_status() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS) + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR); } if self.queue_limits().is_some() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES); } if self.soft_faults() && self.supports_soft_faults() { - flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + flags |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS); } flags } @@ -351,11 +361,7 @@ pub(crate) trait Gpu: Send + Sync { fn base_clock_hz(&self) -> u32; /// Return the GPU address space geometry. fn uat_geometry(&self) -> mmu::UatGeometry; - /// A userspace VM handle was destroyed (VM_DESTROY); release backend-only registry references. - fn vm_destroyed(&self) {} - /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS - /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 - /// T8122 runtime admits any client. + /// Admit a client's parameter ABI before creating its VM. fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { Ok(()) } @@ -380,6 +386,7 @@ pub(crate) trait Gpu: Send + Sync { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result>; /// Bind a BO range as a firmware- or CPU-written timestamp buffer. fn map_timestamp_buffer( @@ -428,8 +435,8 @@ pub(crate) trait GpuManager: Send + Sync { fn get_cfg(&self) -> &'static hw::HwConfig; /// Get the dynamic GPU configuration for this SoC. fn get_dyncfg(&self) -> &hw::DynConfig; - /// Register an unused context as garbage - fn free_context(&self, data: KBox>); + /// Register an unused context, with the objects it retains, for invalidation. + fn free_context(&self, ctx: workqueue::RetiredContext); } /// Private generic trait for functions that don't need to escape this module. @@ -839,6 +846,12 @@ impl GpuManager::ver { let buffer_mgr_clone = buffer_mgr.clone(); let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; let stats_snap_clone = stats_snap.clone(); + // Per-device raw statistics snapshot, shared with the debugfs export. + let raw_stats_snap = Arc::pin_init( + new_spinlock!(stats_raw::RawStatsSnapshot::default(), "asahi_stats_raw"), + GFP_KERNEL, + )?; + let raw_stats_snap_clone = raw_stats_snap.clone(); let alloc_ref = &mut alloc; let rx_channels = KBox::init( try_init!(RxChannels::ver { @@ -854,6 +867,7 @@ impl GpuManager::ver { dev.into(), alloc_ref, stats_snap_clone, + raw_stats_snap_clone, u64::from(cfg.base_clock_hz), )?, }), @@ -881,6 +895,7 @@ impl GpuManager::ver { crashed: AtomicBool::new(false), event_manager, stats: stats_snap.clone(), + raw_stats: raw_stats_snap.clone(), alloc <- new_mutex!(alloc, "alloc"), fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), @@ -1112,7 +1127,7 @@ impl GpuManager::ver { virt_addr: U64(iova + off as u64), total_size: (map.size * map.count * dies) as u32, element_size: map.size as u32, - readwrite: U64(map.writable as u64), + readwrite: U64(u64::from(map.writable)), }; }); @@ -1162,20 +1177,14 @@ impl GpuManager::ver { /// Fetch the GPU MMU fault information from the hardware registers. fn get_fault_info(&self) -> Option { // TODO: handle differently - let Some(guard) = self.dev.registration_guard() else { - return None; - }; + let guard = self.dev.registration_guard()?; guard.registration_data_with(|reg_data| { let res = ®_data.resources; let info = res.get_fault_info(self.cfg); - if info.is_some() { - dev_err!( - self.dev.as_ref(), - " Fault info: {:#x?}\n", - info.as_ref().unwrap() - ); + if let Some(fault) = &info { + dev_err!(self.dev.as_ref(), " Fault info: {:#x?}\n", fault); } info }) @@ -1342,6 +1351,10 @@ impl Gpu for GpuManager::ver { Some(self.stats.clone()) } + fn raw_stats_snapshot(&self) -> Option>> { + Some(self.raw_stats.clone()) + } + fn as_any(&self) -> &dyn Any { self } @@ -1424,6 +1437,7 @@ impl Gpu for GpuManager::ver { ualloc_priv: Arc>, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result> { let mut kalloc = self.alloc(); let id = self.ids.queue.next(); @@ -1441,6 +1455,7 @@ impl Gpu for GpuManager::ver { id, priority, usc_exec_base, + usage, )?, GFP_KERNEL, )?) @@ -1483,7 +1498,7 @@ impl GpuManager for GpuManager::ver { core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); for ctx in garbage_ctx { - if self.invalidate_context(&ctx).is_err() { + if self.invalidate_context(&ctx.data).is_err() { dev_err!( self.dev.as_ref(), "GpuContext: Failed to invalidate GPU context!\n" @@ -1492,6 +1507,9 @@ impl GpuManager for GpuManager::ver { panic!("GPU firmware timed out"); } } + // The firmware no longer knows the context's queues, so their structures may go now, + // ahead of the context data they point to. + core::mem::drop(ctx.retained); } let mut guard = self.alloc.lock(); @@ -1760,7 +1778,7 @@ impl GpuManager for GpuManager::ver { &self.dyncfg } - fn free_context(&self, ctx: KBox>) { + fn free_context(&self, ctx: workqueue::RetiredContext) { let mut garbage = self.garbage_contexts.lock(); if garbage.push(ctx, GFP_KERNEL).is_err() { diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs index 7a4d6f288dc3d..c2e7c06f67b7f 100644 --- a/drivers/gpu/drm/asahi/hw/t6040.rs +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -7,7 +7,7 @@ use crate::g17::fw::{ self, - initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, + initdata::{PerfConfig, RegionCConfig, RegisterWindow}, }; use crate::g17::{Config, CoreLayout, CrashBuffers, FixedBuffer, Identity}; use crate::hw::{GpuGen, GpuVariant}; @@ -62,11 +62,49 @@ pub(crate) const CONFIG: Config = Config { firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { os_version: [26, 6, 2], name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", + images: &[ + crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6040 (normalized loaded image)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0x03, 0xdf, 0x75, 0x48, 0x4a, 0x7f, 0xdd, 0x6e, 0x4b, 0x47, 0x7f, 0x95, 0x6c, + 0xaa, 0x5b, 0x44, 0x06, 0xdf, 0xff, 0x4d, 0x19, 0x17, 0x28, 0x62, 0x7f, 0x53, + 0x10, 0xe0, 0x63, 0x87, 0x6d, 0xcd, + ], + }, + crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6040 (normalized loaded image, alternate J616s fill)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0x19, 0x0c, 0xef, 0x19, 0x46, 0xa2, 0x3d, 0xfb, 0xde, 0x18, 0x05, 0x92, 0x62, + 0x96, 0x31, 0xc3, 0xb2, 0x8d, 0x05, 0x09, 0xa1, 0x7e, 0xb7, 0x61, 0x08, 0xb7, + 0xc9, 0xa5, 0x2a, 0xcc, 0xd9, 0xac, + ], + }, + crate::fwid::FirmwareBuild { + name: "Apple GFX 25G83 for T6040 (normalized loaded image, second alternate J616s fill)", + text_size: 0x60000, + data_size: 0x138000, + entropy: [0x59839, 0x5991c], + text_sha256: [ + 0xd7, 0x25, 0x26, 0x4d, 0x48, 0x31, 0xb9, 0x5a, 0x5b, 0x8a, 0x37, 0x5a, 0xbf, + 0x2f, 0xb5, 0xcd, 0xfb, 0xd6, 0xd7, 0xcb, 0xc7, 0xf1, 0xba, 0xa2, 0xcc, 0x8b, + 0x37, 0xee, 0x5d, 0x6e, 0x5e, 0x98, + ], + }, + ], variant: &fw::variant::V25G83, }]), - // Measured on the 20-core M4 Pro; other core counts are refused at probe. + // Measured on the 20-core M4 Pro; the alternative below remains experimental. core_layout: CoreLayout { cores: Some((2, 20)), + // Admit two clusters of ten slots with eight enabled cores in each cluster. + // Reusing the measured 20-core buffers on it remains experimental. + alternate_core_masks: Some((10, 8)), operand_state: fw::compute::OperandState::TWO_BY_TEN, deflake: fw::render::Deflake::TWO_CLUSTERS, }, @@ -111,7 +149,11 @@ const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> /// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), - instances(window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), 4, 0x200_0000), + instances( + window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), + 4, + 0x200_0000, + ), window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), @@ -120,7 +162,11 @@ pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), - instances(window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), 4, 0x20_0000), + instances( + window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), + 4, + 0x20_0000, + ), window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), @@ -137,33 +183,23 @@ pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), ]; -/// Performance-table configuration. The device tree lists the sixteen operating points in -/// performance-state order, with one voltage per cluster. +/// Inert defaults. The 25G83 builder requires the loader's per-chip performance maps. pub(crate) const PERF: PerfConfig = PerfConfig { - states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], - states_b: [0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15], - relative_a: [0, 9, 16, 23, 29, 38, 49, 60, 69, 79, 100], - relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + states_a: [0; 11], + states_b: [0; 11], + relative_a: [0; 11], + relative_b: [0; 11], sram_min_mv: 810, }; /// Operating points the performance tables reference. pub(crate) const OPPS: usize = 16; -const _: () = { - let mut index = 0; - while index < PERF_STATES { - assert!((PERF.states_a[index] as usize) < OPPS); - assert!((PERF.states_b[index] as usize) < OPPS); - index += 1; - } -}; - /// Region C. const REGION_C: RegionCConfig = RegionCConfig { unk_54: 0x0fa0, unk_78: [1, 0, 0], - max_power_mw: v25g83::MAX_POWER_MW, + max_power_mw: 0, unk_9c: 0, unk_a0: 0, unk_7c4: 1500, diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 75925713bf10b..ea0f9cd782e81 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -57,6 +57,7 @@ pub(crate) const CONFIG: Config = Config { // The layouts the A18 Pro has always run with. core_layout: crate::g17::CoreLayout { cores: None, + alternate_core_masks: None, operand_state: crate::g17::fw::compute::OperandState::ONE_BY_SIX, deflake: crate::g17::fw::render::Deflake::ONE_CLUSTER, }, @@ -417,9 +418,10 @@ pub(crate) mod scheduling { pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; // One physical pair per logical owner leaves room for every execution - // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH. + // context in the shared 128-QID namespace. Commands queue at RENDER_DEPTH, + // as many as a pair has command records (the render memory's RECORDS). pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 1; - pub(crate) const RENDER_DEPTH: u8 = 16; + pub(crate) const RENDER_DEPTH: u8 = 32; pub(crate) const COMPUTE_DEPTH: usize = 16; } diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index bcb9fd32c7d25..3b13ca94462f5 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -250,7 +250,7 @@ impl gpu::Gpu for Backend { } fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, _ualloc_priv: Arc>, priority: u32, - usc_exec_base: u64) -> Result> { + usc_exec_base: u64, _usage: Arc) -> Result> { Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, self.soc.features.fragment_dependency)?,GFP_KERNEL)?) } diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index a60e38bf9addd..1212113af1618 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -44,6 +44,9 @@ pub(crate) struct Packet { pub(crate) wide_visibility:KVec, completion:UserFence, finish_claimed: AtomicBool, + /// Set before a batch with any of this packet's commands can change firmware-visible state, + /// and never cleared. + published: AtomicBool, /// Queue urgency (3 = UAPI realtime .. 0 = UAPI low): more urgent packets are published first. pub(crate) urgency:u8, /// Who submitted the packet, for the failure log only. @@ -79,6 +82,10 @@ impl Packet { } /// Called only after all published commands retired or before any publication. pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} + /// Called before the first firmware-visible change of a batch that includes this packet. + pub(crate) fn mark_published(&self) {self.published.store(true,Ordering::Relaxed);} + /// Whether the firmware may have seen any of this packet's commands. + pub(crate) fn may_be_published(&self)->bool {self.published.load(Ordering::Relaxed)} pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} pub(crate) fn finish(&self,result:Result) { // Retirement, cancellation and a queued scheduler timeout can race. @@ -233,7 +240,7 @@ impl queue::Queue for Queue { parser.finish().map_err(|_|EINVAL)?; let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, completion:self.fences.new_fence(0,Completion)?.into(), - finish_claimed:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), + finish_claimed:AtomicBool::new(false),published:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; let mut job=self.entity.new_job(1,Job{shared:self.shared.clone(),packet})?; for sync in syncs.drain(0..in_sync_count) {if let Some(f)=sync.fence {job.add_dependency(f)?;}} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 5f0ab2b4aea4f..79071cbc434a0 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -71,7 +71,7 @@ pub(crate) fn tlbi_asid(asid: Asid) { asm!( ".arch armv8.4-a", "tlbi aside1os, {x}", - x = in(reg) ((asid as u64) << 48) + x = in(reg) (u64::from(asid) << 48) ); } } @@ -85,7 +85,7 @@ pub(crate) fn tlbi_page(asid: Asid, va: usize) { return; } - let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + let val: u64 = (u64::from(asid) << 48) | ((va as u64 >> 12) & 0xffffffffffc); // SAFETY: tlbi is always safe by definition unsafe { asm!( @@ -147,7 +147,7 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { let bits = 5 * exp; let mut base = (num + (1 << bits) - 1) >> bits; - val |= (exp as u64) << 44; + val |= u64::from(exp) << 44; while base > 32 { // SAFETY: tlbi is always safe by definition diff --git a/drivers/gpu/drm/asahi/memacct.rs b/drivers/gpu/drm/asahi/memacct.rs index 9da09968171e4..12fc90b23d8eb 100644 --- a/drivers/gpu/drm/asahi/memacct.rs +++ b/drivers/gpu/drm/asahi/memacct.rs @@ -131,7 +131,7 @@ fn try_add_within(counter: &AtomicU64, bytes: u64, limit: u64, extra: u64) -> bo fn sub(counter: &AtomicU64, bytes: u64) { let prev = counter.fetch_sub(bytes, Ordering::AcqRel); if prev < bytes { - pr_warn!("asahi: GPU memory accounting underflow ({} < {})\n", prev, bytes); + pr_warn!("GPU memory accounting underflow ({} < {})\n", prev, bytes); counter.store(0, Ordering::Relaxed); } } @@ -157,12 +157,12 @@ impl Ledger { let total = (info.totalram as u64).saturating_mul(u64::from(info.mem_unit)); let limits = limits_for( total, - *module_parameters::mem_limit_pct.value(), - *module_parameters::client_mem_limit_pct.value(), + module_parameters::mem_limit_pct.value(), + module_parameters::client_mem_limit_pct.value(), ); self.set_limits(limits); pr_info!( - "asahi: GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", + "GPU memory limits: device {} MiB, per client {} MiB (0 = none)\n", limits.device >> 20, limits.client >> 20 ); @@ -300,7 +300,7 @@ impl Charge { /// A user object admitted by `commit`, attributed to `mm` if there is one. pub(crate) fn user(commit: Commit, mm: Option>) -> Self { let bytes = commit.bytes; - let mm = if *module_parameters::oom_attribution.value() != 0 { + let mm = if module_parameters::oom_attribution.value() != 0 { mm } else { None diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index cda3ac8d206bd..437685bfad1de 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -108,8 +108,6 @@ pub(crate) use pgtable::{ UAT_TOP_LEVEL_SHIFT, // }; -use pin_init; - const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; @@ -416,6 +414,21 @@ fn lock_vm( Ok(VmExecGuard(unsafe { gpuvm.exec_lock(obj, interruptible)? })) } +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +pub(crate) struct VmReadGuard(VmExecGuard); + +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] +impl VmReadGuard { + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn covers_range(&mut self, address: u64, size: u64, read: bool, write: bool) -> bool { + Vm::covers_range_locked(&mut self.0, address, size, read, write) + } + + pub(crate) fn read_user(&mut self, va: u64, buf: &mut [u8]) -> Result { + Vm::read_user_locked(&mut self.0, va, buf) + } +} + /// Slot binding-related inner data for a Vm instance. struct VmBinding { active_users: usize, @@ -1019,6 +1032,7 @@ impl VmInner { /// Shared reference to a virtual memory address space ([`Vm`]). #[derive(Clone)] pub(crate) struct Vm { + device_binding: Arc, dev: ARef, id: u64, mapping_epoch: Option>, @@ -1229,56 +1243,88 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// Prepared CPU timestamp access; destruction happens after completion signalling. -pub(crate) struct TimestampMapping { - mapping: kernel::drm::gem::shmem::VMapOwned, - offset: usize, -} +/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's +/// reservation lock, so it must not be dropped where fence producers run. +pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); -impl TimestampMapping { - /// Prepare one bounded CPU timestamp destination on retained GEM backing. - pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { +impl TimestampVmap { + pub(crate) fn new(gem: &gem::Object) -> Result { + Ok(Self(gem.owned_vmap::<0>()?)) + } + + fn size(&self) -> usize { use kernel::drm::gem::BaseObject; - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { - return Err(ERANGE); - } - Ok(Self { - mapping: gem.owned_vmap::<0>()?, - offset, - }) + self.0.owner().size() } - pub(crate) fn write(&self, value: u64) { + /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete + /// atomic store; byte stores support unaligned destinations without creating mutable + /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused + /// when the destination is prepared, and ignored here. + pub(crate) fn write(&self, offset: usize, value: u64) { + if offset + .checked_add(size_of::()) + .is_none_or(|end| end > self.size()) + { + return; + } // Completion runs in task context. Bindings of the same BO share this lock, // so aligned u64 and overlapping unaligned byte stores never race in Rust. - let _guard = self.mapping.owner().timestamp_lock.lock(); - // SAFETY: timestamp_mapping checked the complete extent inside this pinned vmap. - let base = self.mapping.addr.cast::(); - let pointer = unsafe { base.add(self.offset) }; + let _guard = self.0.owner().timestamp_lock.lock(); + let base = self.0.addr.cast::(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. + let pointer = unsafe { base.add(offset) }; if pointer as usize % core::mem::align_of::() == 0 { - // SAFETY: timestamp_mapping checked all eight bytes; the vmap pins the backing. + // SAFETY: All eight bytes were checked above; the vmap pins the backing. // A naturally aligned query result becomes available in one complete store. let destination = unsafe { &*pointer.cast::() }; destination.store(value.to_le(), Ordering::Relaxed); } else { for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { &*base.add(self.offset + index).cast::() }; + // SAFETY: The complete eight-byte extent was checked above. The owned + // vmap pins its backing; byte atomics permit unaligned and overlapping + // timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(offset + index).cast::() }; destination.store(byte, Ordering::Relaxed); } } // A CPU-retired query can be consumed by a later GPU command. WRITEBACK // objects use a cacheable vmap, while GPU mappings need not snoop that cache. // Publish the bytes before signalling the completion fence or queue frontier. - if self.mapping.owner().is_writeback() { + if self.0.owner().is_writeback() { crate::pgtable::clean_poc(pointer, size_of::()); } crate::pgtable::poc_barrier(); } } +/// Prepared CPU timestamp access with its own vmap; destruction happens after completion +/// signalling. +#[cfg(CONFIG_DRM_ASAHI_G16)] +pub(crate) struct TimestampMapping { + mapping: TimestampVmap, + offset: usize, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl TimestampMapping { + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + Ok(Self { + mapping: TimestampVmap::new(gem)?, + offset, + }) + } + + pub(crate) fn write(&self, value: u64) { + self.mapping.write(self.offset, value); + } +} + /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. pub(crate) struct KernelMapping(ManuallyDrop>); @@ -1304,16 +1350,22 @@ impl KernelMapping { return Err(EINVAL); } let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - lock_vm(&self.0.owner, None, false)? - .page_table - .translate_iova(address) + self.0.owner.lock_inner().page_table.translate_iova(address) } /// Writes a userspace timestamp through the retained GEM mapping. Aligned results /// use a complete atomic store; byte stores support unaligned destinations without /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + TimestampMapping::new(gem, self.timestamp_offset(offset)?) + } + + /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete + /// extent lies inside both the mapping and the object. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1324,7 +1376,12 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - TimestampMapping::new(gem, start) + Ok(start) + } + + /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. + pub(crate) fn timestamp_vmap(&self) -> Result { + TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1436,7 +1493,7 @@ impl KernelMapping { } else { // Otherwise, check if this slot is the active one, otherwise return // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + let ttb = owner.ttb() | TTBR_VALID | u64::from(slot) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); uat_inner.handoff_lock(); @@ -1679,7 +1736,7 @@ impl UatShared { /// Writes back context-table slot `idx` after it changed, where this SoC needs it. fn publish_slot(&self, idx: usize) { if self.quirks.clean_tables { - let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + let slot = core::ptr::from_ref(&self.ttbs()[idx]).cast::(); pgtable::clean_poc(slot, size_of::()); pgtable::poc_barrier(); } @@ -1688,13 +1745,20 @@ impl UatShared { /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() + unsafe { self.handoff_rgn.map.ptr().cast::().as_ref() }.unwrap() } /// Returns the TTBAT area fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { // SAFETY: pointer is non-null per the type invariant - unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + unsafe { + self.ttbs_rgn + .map + .ptr() + .cast::<[SlotTTBS; UAT_NUM_CTX]>() + .as_ref() + } + .unwrap() } /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. @@ -1882,7 +1946,9 @@ impl Handoff { while self.lock_fw.load(Ordering::Relaxed) != 0 { if self.turn.load(Ordering::Relaxed) != 0 { self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 {} + while self.turn.load(Ordering::Relaxed) != 0 { + core::hint::spin_loop() + } self.lock_ap.store(1, Ordering::Relaxed); fence(Ordering::SeqCst); } @@ -1989,10 +2055,7 @@ impl HandoffFlush { } } -fn prefetch_sgt_segments( - dev: &device::Device, - gem: &gem::Object, -) -> Result { +fn prefetch_sgt_segments(dev: &device::Device, gem: &gem::Object) -> Result { let sgt = crate::gem::held_sg_table(gem, dev)?; let mut segments = KVVec::with_capacity(sgt.iter().size_hint().0, GFP_KERNEL)?; @@ -2000,7 +2063,10 @@ fn prefetch_sgt_segments( segments.push((sgt_entry.dma_address(), sgt_entry.dma_len()), GFP_KERNEL)?; } - Ok(SgSegments { segments, _lease: sgt.into_lease() }) + Ok(SgSegments { + segments, + _lease: sgt.into_lease(), + }) } /// Reuse cached segments without allocating another copy for each bind. The bind lease was @@ -2112,7 +2178,9 @@ impl Vm { )?; let gpuvm = ARef::from(&*unique_gpuvm); + let device_binding = crate::binding::lookup(&dev)?; Ok(Vm { + device_binding, dev: dev.as_ref().as_ref().into(), id, gpuvm, @@ -2144,7 +2212,7 @@ impl Vm { /// Attach a status and job lifetime tracking before sharing a newly created user VM with /// queues, without the G17 canonical context views, shared-object footprints and idle /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn with_tracked_status(mut self) -> Result { self.status = Some(Arc::pin_init( crate::g17::status::VmStatus::new(), @@ -2167,9 +2235,18 @@ impl Vm { /// and invalidated to the point of coherency first (never only invalidated: CPU-written lines /// may be dirty), so the copy sees what the GPU or a write-combined CPU view stored. The /// mappings cannot change during the copy (GPUVM exec lock). - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { - let mut inner = lock_vm(&self.gpuvm, None, false)?; + self.read_lock()?.read_user(va, buf) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + pub(crate) fn read_lock(&self) -> Result { + Ok(VmReadGuard(lock_vm(&self.gpuvm, None, false)?)) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn read_user_locked(inner: &mut VmInner, va: u64, buf: &mut [u8]) -> Result { let mut done = 0usize; while done < buf.len() { let address = va.checked_add(done as u64).ok_or(EFAULT)?; @@ -2239,9 +2316,7 @@ impl Vm { } pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return false; - }; + let mut inner = self.gpuvm.lock_inner(); Self::covers_range_locked(&mut inner, address, size, read, write) } @@ -2256,9 +2331,7 @@ impl Vm { if stride <= block_size { return false; } - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return false; - }; + let mut inner = self.gpuvm.lock_inner(); addresses.iter().all(|&address| { let Some(guard) = address.checked_add(block_size) else { return false; @@ -2268,12 +2341,12 @@ impl Vm { }) } - /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 + /// Checks all exact request tuples under one hold of the VM reservation. Access bit0 /// requests reading and bit1 writing. The epoch names this same snapshot. pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { - let Ok(mut inner) = lock_vm(&self.gpuvm, None, false) else { - return (false, None); - }; + // Every mapping change holds the VM reservation, so readers need only that one. The full + // exec lock would also lock every external BO, making each submit O(external BOs). + let mut inner = self.gpuvm.lock_inner(); let epoch = inner.epoch; let covered = ranges.iter().all(|&(address, size, access)| { Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) @@ -2327,7 +2400,13 @@ impl Vm { guard_size: usize, ) -> Result { self.map_in_range_with_mode( - gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Best, + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Best, ) } @@ -2342,7 +2421,13 @@ impl Vm { guard_size: usize, ) -> Result { self.map_in_range_with_mode( - gem, object_range, alignment, range, prot, guard_size, mm::InsertMode::Low, + gem, + object_range, + alignment, + range, + prot, + guard_size, + mm::InsertMode::Low, ) } @@ -2370,20 +2455,27 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let vm_bo = self - .gpuvm - .obtain(gem, pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; let size = object_range.range(); + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock(/*Some(gem)*/ None, false)? + self.gpuvm.exec_lock_vm(/*Some(gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); @@ -2452,21 +2544,26 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; - let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); - let vm_bo = self - .gpuvm - .obtain(&gem, pin_init!(VmBoData { - newest_bind: AtomicU64::new(0), - segments <- new_mutex!(None), - }))?; - prepare_sgt_segments(dev, gem, &vm_bo, &_bind_lease)?; + // End the bound-device lease before entering firmware/GPUVM waits. + let (_bind_lease, vm_bo) = { + let binding = self.device_binding.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = self.gpuvm.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; // SAFETY: no other UniqRefGpuVm exists so `GpuVm::exec_lock()` is safe let mut vm_exec = unsafe { self.gpuvm - .exec_lock(/*Some(&gem)*/ None, false) + .exec_lock_vm(/*Some(&gem)*/ None, false) .inspect_err(|_| { dev_err!(self.dev, "map_at(): Failed to exec_lock gpuvm\n"); })? @@ -2535,7 +2632,7 @@ impl Vm { gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // Pin and copy backing before taking BO or DMA reservation locks. - let dev = unsafe { self.dev.as_bound() }; + let dev: &device::Device = device.as_ref().as_ref(); let _bind_lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); let vm_bo = self .gpuvm @@ -2556,8 +2653,10 @@ impl Vm { // Mapping needs a complete context let mut ctx = StepContext::new(Some(device.as_ref().as_ref()), prot, &mut resources)?; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. let mut vm_exec = unsafe { - self.gpuvm.exec_lock(/*Some(&gem)*/ None, false)? + self.gpuvm.exec_lock_vm(/*Some(&gem)*/ None, false)? }; let inner = vm_exec.unique_gpuvm.data(); @@ -2628,7 +2727,9 @@ impl Vm { size: usize, prot: Prot, ) -> Result { - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( @@ -2732,7 +2833,9 @@ impl Vm { let mut resources = VmOpResources::for_unmap()?; let mut ctx = StepContext::new(dev, Prot::default(), &mut resources)?; - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + // SAFETY: Vm construction does not retain its initial UniqueRefGpuVm; + // every subsequent access to VM data is serialized by the exec lock. + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; mod_dev_dbg!(self.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); let _commit = if user { @@ -2763,7 +2866,7 @@ impl Vm { let mut ctx = StepContext::new(None, Prot::default(), &mut resources)?; // SAFETY: See `lock_vm()`. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), result => result?, }; @@ -2802,7 +2905,8 @@ impl Vm { // The BO's newest sequence is checked under the same lock as binds. If // it is not newer than the close, every mapping belongs to that close // and the per-object list is safe and substantially cheaper to walk. - let mut vm_exec = match unsafe { self.gpuvm.exec_lock(None, false) } { + // SAFETY: See `lock_vm()`. + let mut vm_exec = match unsafe { self.gpuvm.exec_lock_vm(None, false) } { Err(error) if error == ENOMEM => { return self.drop_mappings_filtered(gem, sequence); } @@ -2942,7 +3046,7 @@ impl Drop for VmInner { /// Physical regions of a UAT whose handoff the generation has already established itself /// ([`Uat::new_adopted`]). None of them needs to be in the kernel linear map. -#[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] +#[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) struct AdoptedRegions { /// The handoff region (PPL magic, lock and flush requests). pub(crate) handoff: Range, @@ -2965,34 +3069,44 @@ impl Uat { }) } - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Map a region given by its physical range, write-combined. - fn map_physical(range: &Range, min_size: usize) -> Result { + fn map_physical( + range: &Range, + min_size: usize, + reserved_ram: bool, + ) -> Result { let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; - if size < min_size || (range.start | range.end) & (UAT_PGMSK as PhysicalAddr) != 0 { + if size < min_size || range.start & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); } // SAFETY: All-zero is a valid `struct resource` (no name, parent or children). let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; raw.start = range.start; raw.end = range.end - 1; - raw.flags = kernel::bindings::IORESOURCE_MEM as _; + raw.flags = kernel::bindings::IORESOURCE_MEM as usize; // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and // `raw` describes a valid memory range. let res: io::resource::Resource = unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; - // A carveout outside System RAM is mapped write-combined; a reserved-memory region - // inside System RAM (mapped by the kernel) through the kernel's own mapping, as - // `map_region` does for the other generations. - let mode = crate::g15::memory_mode(range.start, size as u64)?; - let ram = mode == kernel::bindings::MEMREMAP_WB as u32; - let flags = if ram { - io::mem::MemFlag::WB + let flags = io::mem::MemFlag::WC; + #[cfg(CONFIG_DRM_ASAHI_G15)] + let flags = if reserved_ram { + // G15 admits only whole reserved RAM pages or wholly non-RAM carveouts. + if crate::g15::memory_mode(range.start, size as u64)? + == kernel::bindings::MEMREMAP_WB as u32 + { + io::mem::MemFlag::WB + } else { + flags + } } else { - io::mem::MemFlag::WC + flags }; - // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU - // reaches them through its own translation. + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + let _ = reserved_ram; + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; + // the GPU reaches them through its own translation. let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; Ok(UatRegion { base: range.start, @@ -3000,7 +3114,7 @@ impl Uat { }) } - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Creates a `Uat` for a handoff that the caller establishes itself (G15). /// /// This only maps the handoff and the TTBAT (write-combined); neither is written here. The @@ -3020,25 +3134,36 @@ impl Uat { geometry: UatGeometry, regions: &AdoptedRegions, ) -> Result { - Self::new_adopted_with_user_upper(dev, geometry, regions, true) + Self::new_adopted_inner(dev, geometry, regions, true, true) } /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn new_adopted_with_user_upper( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, regions: &AdoptedRegions, map_kernel_to_user: bool, ) -> Result { - let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; - let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; + Self::new_adopted_inner(dev, geometry, regions, map_kernel_to_user, false) + } + + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] + fn new_adopted_inner( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + map_kernel_to_user: bool, + reserved_ram: bool, + ) -> Result { + let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE, reserved_ram)?; + let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE, reserved_ram)?; // SAFETY: The Handoff struct layout matches the firmware's view of memory at this // address, and the region is at least large enough per the check above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; let inner = Arc::pin_init( try_pin_init!(UatInner { @@ -3110,13 +3235,13 @@ impl Uat { }) } - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Root of the kernel lower `Vm`, for TTBAT context 0 of an adopted handoff. pub(crate) fn kernel_lower_root(&self) -> PhysicalAddr { self.kernel_lower_vm.ttb() } - #[cfg(CONFIG_DRM_ASAHI_UAT_ADOPTION)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] /// Root whose driver-window entry names the second-level table of [`Self::kernel_vm`]'s /// driver window. With [`Self::new_adopted`] this is the shadow root that no context names. pub(crate) fn kernel_window_root(&self) -> PhysicalAddr { @@ -3180,7 +3305,7 @@ impl Uat { return Err(EINVAL); } let root = { - let owner = lock_vm(&vm.gpuvm, None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { return Err(EINVAL); } @@ -3218,6 +3343,16 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, true) + } + + /// Binds a VM without waiting for another slot owner to retire. + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn try_bind(&self, vm: &Vm) -> Result { + self.bind_inner(vm, false) + } + + fn bind_inner(&self, vm: &Vm, wait: bool) -> Result { self.inner.check_m4()?; let mut binding = vm.binding.lock(); self.inner.check_m4()?; @@ -3225,7 +3360,7 @@ impl Uat { if binding.binding.is_none() { assert_eq!(binding.active_users, 0); - let isolation = *module_parameters::robust_isolation.value() != 0; + let isolation = module_parameters::robust_isolation.value() != 0; self.slots.set_limit(if isolation { NonZeroUsize::new(1) @@ -3233,7 +3368,11 @@ impl Uat { None }); - let slot = self.slots.get(binding.bind_token)?; + let slot = if wait { + self.slots.get(binding.bind_token)? + } else { + self.slots.try_get(binding.bind_token)? + }; if slot.changed() { mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); let idx = (slot.slot() as usize) + UAT_USER_CTX_START; @@ -3308,9 +3447,9 @@ impl Uat { { return Err(EIO); } - for index in UAT_USER_CTX_START..UAT_NUM_CTX { - if ttbs[index].ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] - || ttbs[index].ttb1.load(Ordering::Acquire) != 0 + for (index, table) in ttbs.iter().enumerate().skip(UAT_USER_CTX_START) { + if table.ttb0.load(Ordering::Acquire) != inner.owned_user_roots[index] + || table.ttb1.load(Ordering::Acquire) != 0 { return Err(EIO); } @@ -3325,7 +3464,7 @@ impl Uat { return Err(EINVAL); } { - let owner = lock_vm(&vm.gpuvm, None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) { @@ -3411,14 +3550,19 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + UAT_PGSZ / size_of::(), + |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }, + ) }?; mem::sync(); @@ -3431,11 +3575,16 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }) + pgtable::with_root_entries( + self.inner.reserved.as_deref(), + self.kernel_vm.ttb(), + snapshot.len(), + |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }, + ) }?; if !matches { return Err(EIO); @@ -3477,7 +3626,7 @@ impl Uat { } /// Discards the CPU's cached lines of the no-map table regions after the firmware built its - /// root there during boot, before the first host edit (`UatQuirks::reserved_tables`). + /// root there during boot (`UatQuirks::reserved_tables`; see `ReservedTables::invalidate`). pub(crate) fn invalidate_reserved_tables(&self) { if let Some(reserved) = self.inner.reserved.as_ref() { reserved.invalidate(); @@ -3523,7 +3672,7 @@ impl Uat { // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, // and the region is at least large enough per the size specified above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + let handoff = unsafe { &handoff_rgn.map.ptr().cast::().as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); let shared_quirks = quirks; @@ -3652,7 +3801,10 @@ impl Uat { inner.handoff().init(firmware, quirks.handoff_slot)?; if quirks.clean_tables { - pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::clean_poc( + core::ptr::from_ref(inner.handoff()).cast::(), + HANDOFF_SIZE, + ); pgtable::poc_barrier(); } @@ -3680,7 +3832,7 @@ impl Uat { .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } if quirks.clean_tables { - pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::clean_poc(ttbs.as_ptr().cast::(), SLOTS_SIZE); pgtable::poc_barrier(); } @@ -3707,7 +3859,7 @@ impl Vm { /// Diagnostic reservation occupancy (including guards), not a placement promise. /// Removal can progress between queries; allocations are excluded by the VM lock. pub(crate) fn capacity(&self, range: Range) -> Result<(u64, u64)> { - let inner = lock_vm(&self.gpuvm, None, false)?; + let inner = self.gpuvm.lock_inner(); let mut next = range.start; let mut used = 0; let mut largest = 0; diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index e222c7cd8e6a9..acce5677b9f92 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -135,7 +135,7 @@ impl Vm { PreparedUserBindOp::Map(map) => { // SAFETY: See `lock_vm()`. As in `Vm::bind_object()`, the object itself is // not locked: the map step may still populate its pages. - let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; + let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; match self.mapping_commit() { Ok(commit) => { let (flags, repeat) = if map.single_page { @@ -185,7 +185,7 @@ impl Vm { } PreparedUserBindOp::Unmap(unmap) => { // SAFETY: See `lock_vm()`. - let mut inner = unsafe { self.gpuvm.exec_lock(None, true)? }; + let mut inner = unsafe { self.gpuvm.exec_lock_vm(None, true)? }; match self.mapping_commit() { Ok(commit) => { let mut ctx = diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs index 8611221af492a..a63101a813426 100644 --- a/drivers/gpu/drm/asahi/mmu/m3.rs +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -93,7 +93,7 @@ impl Vm { drop(mutation); let _ = inner; ret?; - Ok(KernelMapping(ManuallyDrop::new(node))) + Ok(KernelMapping(node)) } /// [`Vm::map_io`] at `iova`, without the per-mapping log line. diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 49c2241bc399f..1df852184b31d 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -266,7 +266,7 @@ impl VmShrinker { )?; // SAFETY: The format is static and has no format arguments. let raw = - unsafe { kernel::bindings::shrinker_alloc(0, c_str!("asahi-idle-vm").as_char_ptr()) }; + unsafe { kernel::bindings::shrinker_alloc(0, c_str!("aurora-idle-vm").as_char_ptr()) }; if raw.is_null() { return Err(ENOMEM); } diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs index a66a82d92a583..0c9e6f62b5bbe 100644 --- a/drivers/gpu/drm/asahi/object.rs +++ b/drivers/gpu/drm/asahi/object.rs @@ -152,9 +152,9 @@ macro_rules! inner_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; + let inner = p.cast::(); // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; + let off = unsafe { inner.offset_from(outer.cast::()) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -234,9 +234,9 @@ macro_rules! inner_weak_ptr { // we don't actually read from `outer` (which would be UB) nor create an intermediate // reference. let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; + let inner = p.cast::(); // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; + let off = unsafe { inner.offset_from(outer.cast::()) }; // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer // object. unsafe { $gpuva.offset(off.try_into().unwrap(), p) } @@ -298,11 +298,15 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; + let p = alloc + .ptr() + .ok_or(EINVAL)? + .as_ptr() + .cast::>(); let mut raw = callback(&inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is // identical to the type of `raw` other than the lifetime. - unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; + unsafe { p.copy_from(core::ptr::from_mut(&mut raw).cast::().cast(), 1) }; mem::forget(raw); Ok(Self { raw: p, @@ -337,10 +341,14 @@ impl> GpuObject { core::any::type_name::(), alloc.gpu_ptr() ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; + let p = alloc + .ptr() + .ok_or(EINVAL)? + .as_ptr() + .cast::>>(); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. let raw = callback(&inner, unsafe { &mut *p })?; - if p as *mut T::Raw<'_> != raw as *mut _ { + if !core::ptr::eq(p.cast::>(), raw) { dev_err!( alloc.device().as_ref(), "Allocation callback returned a mismatched reference ({})\n", @@ -349,7 +357,7 @@ impl> GpuObject { return Err(EINVAL); } Ok(Self { - raw: p as *mut u8 as *mut T::Raw<'static>, + raw: p.cast::().cast::>(), gpu_ptr, alloc, inner, @@ -400,16 +408,16 @@ impl> GpuObject { alloc.gpu_ptr() ); let inner = inner_init(gpu_ptr); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr().cast::>(); let ret = Self { - raw: p as *mut u8 as *mut T::Raw<'static>, + raw: p.cast::().cast::>(), gpu_ptr, alloc, inner: KBox::init(inner, GFP_KERNEL)?, }; - let q = &*ret.inner as *const T; + let q = core::ptr::from_ref::(&*ret.inner); // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - unsafe { crate::util::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; + unsafe { pin_init::raw_try_init(p, raw_init(&*q, gpu_ptr)) }?; Ok(ret) } @@ -437,7 +445,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } + unsafe { callback(&mut *self.raw, &mut *core::ptr::from_mut(&mut *self.inner)) } } /// Access the inner `Raw` data given a user-supplied callback. @@ -449,7 +457,7 @@ impl> GpuObject { ) -> RetVal { // SAFETY: `self.raw` is valid per the type invariant, and the second half is just // converting lifetimes. - unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } + unsafe { callback(&*self.raw, &*core::ptr::from_ref(&*self.inner)) } } } @@ -475,8 +483,8 @@ where f.debug_struct(core::any::type_name::()) // SAFETY: `self.raw` is valid per the type invariant. .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) - .field("inner", &format_args!("{:#X?}", &self.inner)) - .field("alloc", &format_args!("{:?}", &self.alloc)) + .field("inner", &format_args!("{:#X?}", self.inner)) + .field("alloc", &format_args!("{:?}", self.alloc)) .finish() } } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index c5a47c24ec928..0f915607cb1f4 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -153,9 +153,8 @@ impl ReservedTables { pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { let start = res.start(); // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. - let map = unsafe { - kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? - }; + let map = + unsafe { kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? }; self.starts.push(start, GFP_KERNEL)?; self.regions.push(map, GFP_KERNEL)?; Ok(()) @@ -183,6 +182,15 @@ impl ReservedTables { /// Drops the CPU's cached lines of every region, so that later reads see what the firmware /// wrote while booting. Lines the host dirtied are written back first, never discarded. + /// + /// This runs once, after the firmware booted. The host has written table entries by then + /// (fixed kernel objects are mapped before the boot), but the T6040, the only SoC with + /// reserved tables, also sets `UatQuirks::clean_tables`: every host table store is cleaned + /// to the PoC as it is made, so this drops only clean lines and loses no host write. + /// + /// The regions are not invalidated again, which assumes the firmware no longer writes the + /// tables the host walks once it has booted. That is not proven by a trace; the J616s has + /// run this way since its GPU came up without a fault traced to a stale table. pub(crate) fn invalidate(&self) { for map in self.regions.iter() { clean_invalidate_poc(map.ptr(), map.size()); @@ -192,7 +200,9 @@ impl ReservedTables { } /// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the -/// reserved mapping when the page lies in a reserved region, else through `struct page`. +/// reserved mapping when the page lies in a reserved region, else through `struct page`. With +/// reserved regions, a page outside them that is not System RAM with a `struct page` (other +/// no-map memory) is refused with `EIO`. /// /// # Safety /// @@ -211,8 +221,14 @@ unsafe fn with_table( // SAFETY: lookup() checked that the whole page is mapped. return f(unsafe { base.add(offset) }); } - // SAFETY: The caller guarantees an allocated page-table page in RAM. - let page = unsafe { Page::borrow_phys_unchecked(&phys) }; + let page = if reserved.is_some() { + // SAFETY: The caller guarantees the page stays allocated; borrow_phys() checks that it + // is System RAM with a struct page. + unsafe { Page::borrow_phys(&phys) }.ok_or(EIO)? + } else { + // SAFETY: The caller guarantees an allocated page-table page in RAM. + unsafe { Page::borrow_phys_unchecked(&phys) } + }; page.with_pointer_into_page(offset, len, f) } @@ -353,7 +369,7 @@ impl Prot { /// The protection bits of a leaf with this protection (no address, table type or non-global /// bits), for checking that a firmware leaf word round-trips through [`Self::from_pte`]. - #[cfg(CONFIG_DRM_ASAHI_G15)] + #[cfg(any(CONFIG_DRM_ASAHI_G15, CONFIG_DRM_ASAHI_G16))] pub(crate) const fn leaf_bits(&self) -> u64 { self.as_pte() } @@ -654,7 +670,7 @@ impl UatPageTable { let driver_tables = &mut self.driver_tables; let oas_mask = self.oas_mask; let entry = |p: *mut u8| -> Result> { - let uptep = p as *const _ as *const Pte; + let uptep = p.cast_const().cast::(); // SAFETY: with_table() ensures the pointer is valid, and our index is // aligned so it is safe to deref as an AtomicU64. let upte = unsafe { &*uptep }; @@ -662,7 +678,10 @@ impl UatPageTable { // Allocate if requested if upte_val == 0 && alloc { let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + mod_pr_debug!( + "UATPageTable::with_pages: alloc PT at {:#x}\n", + pt_page.phys() + ); if clean { // A non-snooping walker must never follow a parent to a // table whose zeroes are still in the CPU caches. @@ -738,7 +757,7 @@ impl UatPageTable { iova ); let leaves = |p: *mut u8| -> Result { - let ptep = p as *const _ as *const Pte; + let ptep = p.cast_const().cast::(); // SAFETY: We know this is a valid pointer to PTEs and the range is valid and // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; @@ -752,7 +771,13 @@ impl UatPageTable { // SAFETY: Page table addresses are either allocated by us, or // firmware-managed tables in RAM or in a reserved region. unsafe { - with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + with_table( + reserved.as_deref(), + phys, + idx * PTE_SIZE, + count * PTE_SIZE, + leaves, + ) }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); @@ -789,7 +814,10 @@ impl UatPageTable { } if let Some((parent_phys, parent_index)) = parent { let oas_mask = self.oas_mask; - let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + let in_reserved = self + .reserved + .as_ref() + .is_some_and(|r| r.contains(parent_phys)); // SAFETY: Only looks the page up; nothing is accessed through it. if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { return Err(EIO); @@ -815,7 +843,15 @@ impl UatPageTable { // its child descriptor is detached. A changed descriptor indicates // an ownership violation; fail closed instead of freeing a table // that may still be reachable. - unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; + unsafe { + with_table( + reserved, + parent_phys, + parent_index * PTE_SIZE, + PTE_SIZE, + detach, + ) + }?; } if let Some(tables) = self.driver_tables.as_mut() { let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 4abdd3e485f72..aa5dc1fba4723 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -67,7 +67,9 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { - if layouts != [0; 2] { return Err(EINVAL); } + if layouts != [0; 2] { + return Err(EINVAL); + } Ok(()) } @@ -151,6 +153,7 @@ pub(crate) struct QueueInner { notifier_list: Arc>, notifier: Arc>, usc_exec_base: u64, + usage: Arc, id: u64, #[ver(V >= V13_0B4)] counter: AtomicU64, @@ -191,7 +194,7 @@ impl JobFence::ver { #[vtable] impl dma_fence::FenceOps for JobFence::ver { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") + c_str!("aurora") } fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("queue") @@ -263,10 +266,13 @@ impl QueueJob::ver { impl sched::JobImpl for QueueJob::ver { type TimeoutData = QueueTimeout; const MODULE: Option<&'static kernel::ThisModule> = - Some(&crate::THIS_MODULE); + Some(kernel::module::this_module::()); fn timeout_data(&self) -> QueueTimeout { - QueueTimeout { dev: self.dev.clone(), id: self.id } + QueueTimeout { + dev: self.dev.clone(), + id: self.id, + } } fn prepare(job: &mut Self) -> Option { @@ -358,7 +364,7 @@ impl sched::JobImpl for QueueJob::ver { return Err(ENODEV); }; - return guard.registration_data_with(|reg_data| { + guard.registration_data_with(|reg_data| { let gpu = match reg_data .gpu .clone() @@ -433,7 +439,7 @@ impl sched::JobImpl for QueueJob::ver { core::mem::drop(frag_job); Ok(Some(Fence::from_fence(&job.fence))) - }); + }) } fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { @@ -462,7 +468,7 @@ impl Drop for QueueJob::ver { } } -static QUEUE_NAME: &CStr = c_str!("asahi_fence"); +static QUEUE_NAME: &CStr = c_str!("aurora_fence"); static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); #[versions(AGX)] @@ -482,6 +488,7 @@ impl Queue::ver { id: u64, priority: u32, usc_exec_base: u64, + usage: Arc, ) -> Result { mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); @@ -519,12 +526,21 @@ impl Queue::ver { WQ_SIZE, 0, 100000, - c_str!("asahi_sched"), + c_str!("aurora_sched"), )?; let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; let buffer = buffer::Buffer::ver::new(gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; + let gpu_context = Arc::new( + workqueue::GpuContext::new(dev.clone(), alloc, buffer.any_ref())?, + GFP_KERNEL, + )?; + // Every command of this queue points the firmware at the notifier, and the notifier links + // into the notifier list of the work queues. Keep it as long as they are kept, until the + // context is invalidated, instead of freeing it with the last command. + gpu_context.retain(KBox::new(notifier.clone(), GFP_KERNEL)?)?; + let mut ret = Queue::ver { dev: dev.clone(), _sched: sched, @@ -538,15 +554,12 @@ impl Queue::ver { inner: QueueInner::ver { dev: dev.clone(), ualloc, - gpu_context: Arc::new( - workqueue::GpuContext::new(dev.clone(), alloc, buffer.any_ref())?, - GFP_KERNEL, - )?, - + gpu_context, buffer, notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, notifier, usc_exec_base, + usage, id, #[ver(V >= V13_0B4)] counter: AtomicU64::new(0), @@ -554,7 +567,7 @@ impl Queue::ver { }; // Rendering structures - let tvb_blocks = *module_parameters::initial_tvb_size.value(); + let tvb_blocks = module_parameters::initial_tvb_size.value(); ret.inner.buffer.ensure_blocks(tvb_blocks)?; @@ -624,8 +637,11 @@ const COMPUTE_SUPPORTED_SIZE: usize = // SAFETY: All bit patterns are valid by construction. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_compute {} +// SAFETY: This UAPI record consists of integer fields and nested integer records. unsafe impl AnyBitPattern for uapi::drm_asahi_attachment {} fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { @@ -636,8 +652,10 @@ fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result, size: usize) -> Result> 7; - attachments.list[i as usize] = microseq::Attachment { + attachments.list[i] = microseq::Attachment { address: U64(att.pointer), size: cache_lines.try_into()?, unk_c: 0x17, @@ -756,16 +774,24 @@ impl Queue for Queue::ver { while !cmdbuf.is_empty() { let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; let payload_size = header.size as usize; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(payload_size)?; - if render.ts_frag_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + if render + .ts_frag_end_copies + .iter() + .any(|ts| ts.handle != 0 || ts.offset != 0) + { return Err(EINVAL); } } uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(payload_size)?; - if compute.ts_end_copies.iter().any(|ts| ts.handle != 0 || ts.offset != 0) { + if compute + .ts_end_copies + .iter() + .any(|ts| ts.handle != 0 || ts.offset != 0) + { return Err(EINVAL); } } @@ -773,7 +799,7 @@ impl Queue for Queue::ver { } nr_commands += 1; - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { last_render = nr_commands; nr_render += 1; @@ -865,7 +891,7 @@ impl Queue for Queue::ver { EINVAL })? { let mut alloc = gpu.alloc(); - let queue_job = match header.cmd_type as u32 { + let queue_job = match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, _ => return Err(EINVAL), @@ -895,7 +921,7 @@ impl Queue for Queue::ver { } } - match header.cmd_type as u32 { + match u32::from(header.cmd_type) { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { let render: uapi::drm_asahi_cmd_render = cmdbuf .read_supported(header_size, RENDER_SUPPORTED_SIZE) diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 0a4ee8e7563ec..e8b845cba989f 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -17,9 +17,11 @@ use kernel::{ c_str, device::Core, error::{from_err_ptr, to_result}, - devres::Devres, io::{ - mem::IoMem, + mem::{ + DevresIoMem, + IoMem, // + }, Io, IoBase, // }, @@ -564,7 +566,7 @@ impl Resources<'_> { #[derive(Clone)] pub(crate) struct G17Resources { dev: ARef, - sgx: Arc>>, + sgx: Arc>, } impl G17Resources { @@ -707,6 +709,11 @@ impl G17Resources { self.sgx_read32::() & 0xf != 0 } + /// Raw core power state, for tracing only. + pub(crate) fn g17_core_power(&self) -> u32 { + self.sgx_read32::() + } + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { @@ -775,7 +782,7 @@ fn gpu_revision( #[cfg(CONFIG_DRM_ASAHI_G16)] #[derive(Clone)] pub(crate) struct G16Resources { - sgx: Arc>>, + sgx: Arc>, } #[cfg(CONFIG_DRM_ASAHI_G16)] diff --git a/drivers/gpu/drm/asahi/sysfs.c b/drivers/gpu/drm/asahi/sysfs.c index 7d7d785a83133..7132e6de41c09 100644 --- a/drivers/gpu/drm/asahi/sysfs.c +++ b/drivers/gpu/drm/asahi/sysfs.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -16,7 +18,7 @@ #include "sysfs.h" -/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ +/* Legacy prefix of repr(C) Rust atomics; READ_ONCE accesses current values. */ struct asahi_stats_snapshot { u32 util1; u32 util2; @@ -48,8 +50,69 @@ struct asahi_stats_attribute { const struct asahi_stats_snapshot *snapshot; const struct asahi_gpu_memory *memory; bool export_enabled; + asahi_stats_read_fn read; + asahi_stats_suspend_fn suspend; + struct notifier_block pm_notifier; }; +/* 0: unavailable, 1: valid, 2: stale; matches g17::telemetry. */ +static const char *asahi_stats_quality(u32 status) +{ + return status == 1 ? "valid" : status == 2 ? "stale" : "unavailable"; +} + +static ssize_t agx_qualified_stats_show(struct asahi_stats_attribute *stats, char *buf) +{ + struct asahi_qualified_stats snap; + ssize_t n = 0; + + stats->read(stats->snapshot, &snap); + n += sysfs_emit_at(buf, n, "utilisation_status %s\n", + asahi_stats_quality(snap.util_status)); + if (snap.util_status == 1) { + n += sysfs_emit_at(buf, n, "util_tiler_percent %u\n", snap.tiler); + n += sysfs_emit_at(buf, n, "util_renderer_percent %u\n", snap.renderer); + n += sysfs_emit_at(buf, n, "util_device_percent %u\n", snap.device); + } else { + n += sysfs_emit_at(buf, n, "util_tiler_percent unsupported\n" + "util_renderer_percent unsupported\n" + "util_device_percent unsupported\n"); + } + n += sysfs_emit_at(buf, n, "pstate_status %s\n", + asahi_stats_quality(snap.pstate_status)); + if (snap.pstate_status == 1) + n += sysfs_emit_at(buf, n, "pstate %u\n", snap.pstate); + else + n += sysfs_emit_at(buf, n, "pstate unsupported\n"); + /* No A/B selector, packet duration, measured power or temperature. */ + n += sysfs_emit_at(buf, n, "requested_frequency_hz unsupported\n" + "achieved_frequency_hz unsupported\n" + "busy_ns unsupported\n" + "power_mw unsupported\n" + "temperature_raw unsupported\n"); + return n; +} + +static int asahi_stats_pm_notify(struct notifier_block *nb, unsigned long action, void *unused) +{ + struct asahi_stats_attribute *stats = + container_of(nb, struct asahi_stats_attribute, pm_notifier); + + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + stats->suspend(stats->snapshot, true); + break; + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + stats->suspend(stats->snapshot, false); + break; + } + return NOTIFY_OK; +} + static ssize_t agx_stats_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -60,6 +123,8 @@ static ssize_t agx_stats_show(struct device *dev, if (!stats->export_enabled) return sysfs_emit(buf, "unsupported\n"); + if (stats->read) + return agx_qualified_stats_show(stats, buf); n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", (unsigned long long)READ_ONCE(snap->busy_ns)); n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", @@ -118,14 +183,18 @@ static ssize_t gpu_memory_show(struct device *dev, * registered. Returns 0 on success, or a negative errno. */ int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle) + int export_enabled, const void *memory, void **handle, + asahi_stats_read_fn read, asahi_stats_suspend_fn suspend) { struct asahi_stats_attribute *stats; int ret; - if (!dev || !handle || !memory || (export_enabled && !snapshot)) + if (!dev || !handle || !memory || ((export_enabled || read) && !snapshot) || + (!!read != !!suspend)) return -ENODEV; + BUILD_BUG_ON(offsetof(struct asahi_qualified_stats, util_host_ns) != 24); + BUILD_BUG_ON(sizeof(struct asahi_qualified_stats) != 56); BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, refused) != 24); BUILD_BUG_ON(offsetof(struct asahi_gpu_memory, client_limit) != 40); BUILD_BUG_ON(sizeof(struct asahi_gpu_memory) != 48); @@ -145,6 +214,16 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, stats->attr.show = agx_stats_show; stats->snapshot = snapshot; stats->export_enabled = export_enabled; + stats->read = read; + stats->suspend = suspend; + if (suspend) { + stats->pm_notifier.notifier_call = asahi_stats_pm_notify; + ret = register_pm_notifier(&stats->pm_notifier); + if (ret) { + kfree(stats); + return ret; + } + } sysfs_attr_init(&stats->memory_attr.attr); stats->memory_attr.attr.name = "gpu_memory"; @@ -154,6 +233,8 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, ret = device_create_file(dev, &stats->attr); if (ret) { + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -161,6 +242,8 @@ int asahi_sysfs_register(struct device *dev, const void *snapshot, ret = device_create_file(dev, &stats->memory_attr); if (ret) { device_remove_file(dev, &stats->attr); + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); kfree(stats); return ret; } @@ -217,6 +300,9 @@ void asahi_sysfs_unregister(struct device *dev, void *handle) if (!dev || !stats) return; + /* Both notifier removal and file removal join readers of the Rust Arc. */ + if (stats->suspend) + unregister_pm_notifier(&stats->pm_notifier); /* Removal drains active show callbacks before either allocation dies. */ device_remove_file(dev, &stats->memory_attr); device_remove_file(dev, &stats->attr); @@ -230,4 +316,4 @@ EXPORT_SYMBOL_GPL(asahi_sysfs_unregister); MODULE_AUTHOR("AGX driver maintainer"); MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("Asahi AGX firmware stats sysfs shim"); +MODULE_DESCRIPTION("AGX firmware stats sysfs shim"); diff --git a/drivers/gpu/drm/asahi/sysfs.h b/drivers/gpu/drm/asahi/sysfs.h index f1873f9be6caa..d43f482903e55 100644 --- a/drivers/gpu/drm/asahi/sysfs.h +++ b/drivers/gpu/drm/asahi/sysfs.h @@ -3,10 +3,30 @@ #ifndef _ASAHI_SYSFS_H #define _ASAHI_SYSFS_H +#include + struct device; +/* Mirrors g17::telemetry::Readout, filled under the Rust device mutex. */ +struct asahi_qualified_stats { + u32 util_status; + u32 pstate_status; + u32 tiler; + u32 renderer; + u32 device; + u32 pstate; + u64 util_host_ns; + u64 pstate_host_ns; + u64 util_samples; + u64 pstate_timestamp; +}; + +typedef void (*asahi_stats_read_fn)(const void *snapshot, struct asahi_qualified_stats *out); +typedef void (*asahi_stats_suspend_fn)(const void *snapshot, bool suspended); + int asahi_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, const void *memory, void **handle); + int export_enabled, const void *memory, void **handle, + asahi_stats_read_fn read, asahi_stats_suspend_fn suspend); int asahi_sysfs_register_idle(struct device *dev, u32 *word); void asahi_sysfs_unregister(struct device *dev, void *handle); diff --git a/drivers/gpu/drm/asahi/sysfs_exports.rs b/drivers/gpu/drm/asahi/sysfs_exports.rs index cee5d52a4e261..7c34f0627b020 100644 --- a/drivers/gpu/drm/asahi/sysfs_exports.rs +++ b/drivers/gpu/drm/asahi/sysfs_exports.rs @@ -3,16 +3,21 @@ //! Per-device sysfs registration. The caller retains the snapshot Arc until //! unregister has drained active readers and freed the C attribute wrapper. -use crate::{memacct::Ledger, stats::StatsSnapshot}; +use crate::{g17::telemetry::Readout, memacct::Ledger, stats::StatsSnapshot}; use core::ffi::{c_int, c_void}; +type ReadStats = unsafe extern "C" fn(*const c_void, *mut Readout); +type SuspendStats = unsafe extern "C" fn(*const c_void, bool); + extern "C" { fn asahi_sysfs_register( dev: *mut c_void, - snapshot: *const StatsSnapshot, + snapshot: *const c_void, export_enabled: c_int, memory: *const Ledger, handle: *mut *mut c_void, + read: Option, + suspend: Option, ) -> c_int; fn asahi_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); } @@ -29,7 +34,11 @@ pub(crate) fn register_idle( // SAFETY: `word` is a static atomic that outlives the attribute; the C side reads and writes // it with READ_ONCE/WRITE_ONCE. let ret = unsafe { asahi_sysfs_register_idle(dev.cast(), word) }; - if ret < 0 { Err(kernel::error::Error::from_errno(ret)) } else { Ok(()) } + if ret < 0 { + Err(kernel::error::Error::from_errno(ret)) + } else { + Ok(()) + } } /// Register only after initialization succeeds, keeping the snapshot Arc alive. @@ -39,6 +48,66 @@ pub(crate) fn register( dev: *mut kernel::bindings::device, snapshot: *const StatsSnapshot, export_enabled: bool, +) -> kernel::error::Result<*mut c_void> { + register_with_callbacks(dev, snapshot, export_enabled, None, None) +} + +/// The caller retains the G17 snapshot Arc until unregister joins both sysfs +/// readers and the PM notifier. The callbacks never power the GPU up. +pub(crate) fn register_g17( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: bool, +) -> kernel::error::Result<*mut c_void> { + register_with_callbacks( + dev, + snapshot, + export_enabled, + Some(read_g17), + Some(suspend_g17), + ) +} + +/// Copy the locked qualified snapshot to the C formatter's local output. +/// +/// # Safety +/// +/// `snapshot` must name a live StatsSnapshot retained by the registration. +/// `out` must be aligned and writable for one Readout for this call. +unsafe extern "C" fn read_g17(snapshot: *const c_void, out: *mut Readout) { + // SAFETY: C calls only while registration retains the snapshot Arc, and + // supplies a writable, aligned output for this call. + let snapshot = unsafe { &*snapshot.cast::() }; + let view = snapshot + .g17_reader + .as_ref() + .map_or_else(Readout::default, |reader| reader.read()); + // SAFETY: As above; the C output is initialized in one locked read. + unsafe { out.write(view) }; +} + +/// End the telemetry epoch at either side of system sleep. +/// +/// # Safety +/// +/// `snapshot` must name a live StatsSnapshot. The caller must join this +/// callback before releasing the registration's owning Arc. +unsafe extern "C" fn suspend_g17(snapshot: *const c_void, suspended: bool) { + // SAFETY: The notifier is unregistered and joined before the Arc is dropped. + if let Some(reader) = unsafe { &*snapshot.cast::() } + .g17_reader + .as_ref() + { + reader.suspend(suspended); + } +} + +fn register_with_callbacks( + dev: *mut kernel::bindings::device, + snapshot: *const StatsSnapshot, + export_enabled: bool, + read: Option, + suspend: Option, ) -> kernel::error::Result<*mut c_void> { let mut handle = core::ptr::null_mut(); // SAFETY: The bound driver keeps the device and optional snapshot alive until @@ -46,10 +115,12 @@ pub(crate) fn register( let ret = unsafe { asahi_sysfs_register( dev.cast(), - snapshot, + snapshot.cast(), c_int::from(export_enabled), &crate::memacct::LEDGER, &mut handle, + read, + suspend, ) }; if ret < 0 { diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 976ec9f6bf48b..9f13436b4caf5 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -13,10 +13,20 @@ use kernel::{prelude::*, sync::Arc}; #[cfg(CONFIG_DRM_ASAHI_G16)] use kernel::{uapi, xarray}; +/// A firmware-mapped timestamp buffer with a kernel CPU mapping of its backing, kept for the +/// buffer's lifetime so that commands need no mapping of their own. +pub(crate) struct FirmwareBuffer { + mapping: Arc, + cpu: mmu::TimestampVmap, +} + /// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. /// CPU-retired timestamps retain their backing without consuming global firmware address space. pub(crate) enum Buffer { - Firmware(Arc), + /// The last reference may be a command's, dropped where fence producers run, while + /// releasing the CPU mapping takes the reservation lock: the deferral runs the release + /// on the cleanup queue. + Firmware(crate::cleanup::Deferred), #[cfg(CONFIG_DRM_ASAHI_G16)] Cpu { // The GEM pin keeps the object's held SG table alive through CPU publication. @@ -27,7 +37,11 @@ pub(crate) enum Buffer { impl Buffer { pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { - Ok(Self::Firmware(Arc::new(mapping, GFP_KERNEL)?)) + let cpu = mapping.timestamp_vmap()?; + Ok(Self::Firmware(crate::cleanup::Deferred::new(FirmwareBuffer { + mapping: Arc::new(mapping, GFP_KERNEL)?, + cpu, + })?)) } /// Retain a CPU-written byte range without publishing any firmware address. @@ -46,15 +60,16 @@ impl Buffer { pub(crate) fn size(&self) -> usize { match self { - Self::Firmware(mapping) => mapping.size(), + Self::Firmware(buffer) => buffer.mapping.size(), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { range, .. } => range.end - range.start, } } + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { match self { - Self::Firmware(mapping) => mapping.timestamp_mapping(offset), + Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { gem, range, .. } => { if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -65,10 +80,30 @@ impl Buffer { } } + /// The object offset of the timestamp at `offset` of a firmware buffer, for + /// [`Self::write_timestamp`]. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { + match self { + Self::Firmware(buffer) => buffer.mapping.timestamp_offset(offset), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => Err(EINVAL), + } + } + + /// Writes a timestamp at an object offset returned by [`Self::timestamp_offset`], + /// through the buffer's own CPU mapping. + pub(crate) fn write_timestamp(&self, offset: usize, value: u64) { + match self { + Self::Firmware(buffer) => buffer.cpu.write(offset, value), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => {} + } + } + /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. pub(crate) fn firmware_mapping(&self) -> Result> { match self { - Self::Firmware(mapping) => Ok(mapping.clone()), + Self::Firmware(buffer) => Ok(buffer.mapping.clone()), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { .. } => Err(EINVAL), } diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index a5f9455b23f5f..6e5540caeac3f 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -96,6 +96,13 @@ pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { m } +/// A value that can be populated from an untrusted byte record. +/// +/// # Safety +/// +/// Every bit pattern must be valid for Self. Its byte representation must +/// contain no uninitialized bytes (including padding) when produced by Default, +/// since Reader temporarily views that storage as a byte slice. pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} pub(crate) struct Reader<'a> { @@ -117,7 +124,9 @@ impl<'a> Reader<'a> { // SAFETY: The output pointer is valid, and the size does not exceed // the type size, and all bit patterns are valid. - let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; + let dst = unsafe { + core::slice::from_raw_parts_mut(core::ptr::from_mut(&mut obj).cast::(), size) + }; dst.copy_from_slice(&src[..size]); self.offset = end; @@ -217,17 +226,3 @@ mod tests { assert!(reader.read_supported::(8, 8).is_err()); } } - -/// Fallibly initializes `slot` with `init` (`pin_init::raw_try_init` of later pin-init releases). -/// -/// # Safety -/// -/// - `slot` is a valid pointer to uninitialized memory. -/// - the caller does not touch `slot` when `Err` is returned, they are only permitted to -/// deallocate. -/// - `slot` will not move until it is dropped, i.e. it will be pinned. -#[inline(always)] -pub(crate) unsafe fn raw_try_init(slot: *mut T, init: impl PinInit) -> Result<(), E> { - // SAFETY: Per safety requirement. - unsafe { init.__pinned_init(slot) } -} diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 9de2e7f0f4f97..4b613b1cc7f7a 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -491,10 +491,23 @@ void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, } EXPORT_SYMBOL_GPL(drm_gem_shmem_vunmap_locked); -static int -drm_gem_shmem_create_with_handle(struct drm_file *file_priv, - struct drm_device *dev, size_t size, - uint32_t *handle) +/** + * drm_gem_shmem_create_with_handle - Allocate an object with the given size and + * returns a GEM handle + * @file_priv: DRM file structure to create the dumb buffer for + * @dev: DRM device + * @size: Size of the object to allocate + * @handle: Returns the GEM handle on success + * + * Allocates an shmem GEM buffer using drm_gem_shmem_create() and returns + * a GEM handle to it. + * + * Returns: + * Zero on success, or an error code otherwise. + */ +int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + uint32_t *handle) { struct drm_gem_shmem_object *shmem; int ret; @@ -513,6 +526,7 @@ drm_gem_shmem_create_with_handle(struct drm_file *file_priv, return ret; } +EXPORT_SYMBOL_GPL(drm_gem_shmem_create_with_handle); /* Update madvise status, returns true if not purged, else * false or -errno. @@ -988,7 +1002,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; } - if (is_cow_mapping(vma->vm_flags)) + if (vma_is_cow_mapping(vma)) return -EINVAL; #ifdef CONFIG_MM_SUBPAGE @@ -1023,7 +1037,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return ret; vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); if (shmem->map_wc) vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h index 30791d328550d..8392f9e4a0109 100644 --- a/include/drm/drm_gem_shmem_helper.h +++ b/include/drm/drm_gem_shmem_helper.h @@ -168,6 +168,10 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem) void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem, struct drm_printer *p, unsigned int indent); +int drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + uint32_t *handle); + extern const struct vm_operations_struct drm_gem_shmem_vm_ops; /* diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 66396f89cc333..418522205ee6f 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -368,6 +368,60 @@ enum drm_asahi_feature { * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. */ DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, + + /* + * Bits 17 to 28 belong to the M5 (G17G/G17S) runtime, which reports + * them; this kernel never does. They are listed so that the numbers are + * not given to anything else, and userspace reads every kernel's + * features the same way. Bits 25 and 27 are also reserved for that + * runtime. + */ + + /** @DRM_ASAHI_FEATURE_RENDER_RESOURCES: M5: per-program render storage. */ + DRM_ASAHI_FEATURE_RENDER_RESOURCES = (1UL) << 17, + + /** @DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2: M5: versioned render storage. */ + DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2 = (1UL) << 18, + + /** @DRM_ASAHI_FEATURE_SPARSE_TILES: M5: DRM_ASAHI_BIND_SPARSE_TILE. */ + DRM_ASAHI_FEATURE_SPARSE_TILES = (1UL) << 19, + + /** @DRM_ASAHI_FEATURE_COMPUTE_MATRIX: M5: compute matrix instructions. */ + DRM_ASAHI_FEATURE_COMPUTE_MATRIX = (1UL) << 20, + + /** @DRM_ASAHI_FEATURE_TASK_MESH_STORAGE: M5: task and mesh storage. */ + DRM_ASAHI_FEATURE_TASK_MESH_STORAGE = (1UL) << 21, + + /** @DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES: M5: per-render shader writes. */ + DRM_ASAHI_FEATURE_RENDER_SHADER_WRITES = (1UL) << 22, + + /** + * @DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION: M5: compressed + * multisample depth. + */ + DRM_ASAHI_FEATURE_M5_DEPTH_MSAA_COMPRESSION = (1UL) << 23, + + /** + * @DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS: M5: a compute prefix for a + * render's tiling half. + */ + DRM_ASAHI_FEATURE_COMPUTE_STAGE_BARRIERS = (1UL) << 24, + + /** + * @DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION: M5: compressed + * multisample stencil. + */ + DRM_ASAHI_FEATURE_M5_STENCIL_MSAA_COMPRESSION = (1UL) << 26, + + /** @DRM_ASAHI_FEATURE_COMPUTE_RESOURCES: M5: per-dispatch storage. */ + DRM_ASAHI_FEATURE_COMPUTE_RESOURCES = (1UL) << 28, + + /** + * @DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS: SUBMIT accepts one + * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs + * include the earlier points of that timeline without adding GPU waits. + */ + DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -391,7 +445,10 @@ enum drm_asahi_feature { * The driver publishes an accepted job's failure before signaling its fence. * Thus userspace must query after a successful DRM syncobj wait/poll before * treating that fence as successful execution. Syncobj signal alone is not - * an execution-success indication. + * an execution-success indication. A job cancelled (ECANCELED) because + * another VM's recovery discarded it, an input did not signal in time, or + * its queue was closed, is not a failure of this VM: its fence carries the + * error, error stays 0, and its output is undefined. * The feature bit is the availability contract; older kernels leave it zero. * Kernel errors for invalid requests: EINVAL flags/pad/size, ENOENT foreign or * missing vm_id, EFAULT inaccessible pointer. No output on invalid request. @@ -646,6 +703,12 @@ enum drm_asahi_bind_flags { * It is intended as a mechanism to accelerate sparse. */ DRM_ASAHI_BIND_SINGLE_PAGE = (1L << 3), + + /** + * @DRM_ASAHI_BIND_SPARSE_TILE: Reserved for the M5 runtime + * (DRM_ASAHI_FEATURE_SPARSE_TILES). This kernel rejects it. + */ + DRM_ASAHI_BIND_SPARSE_TILE = (1L << 4), }; /** @@ -807,6 +870,24 @@ enum drm_asahi_cmd_type { * Followed by (possibly multiple) @drm_asahi_attachment payloads. */ DRM_ASAHI_SET_COMPUTE_ATTACHMENTS = 4, + + /** + * @DRM_ASAHI_SET_RENDER_RESOURCES: Reserved for the M5 runtime + * (DRM_ASAHI_FEATURE_RENDER_RESOURCES). This kernel rejects it. + */ + DRM_ASAHI_SET_RENDER_RESOURCES = 5, + + /** + * @DRM_ASAHI_SET_COMPUTE_RESOURCES: Reserved for the M5 runtime + * (DRM_ASAHI_FEATURE_COMPUTE_RESOURCES). This kernel rejects it. + */ + DRM_ASAHI_SET_COMPUTE_RESOURCES = 6, + + /** + * @DRM_ASAHI_SET_RENDER_RESOURCES_V2: Reserved for the M5 runtime + * (DRM_ASAHI_FEATURE_RENDER_RESOURCES_V2). This kernel rejects it. + */ + DRM_ASAHI_SET_RENDER_RESOURCES_V2 = 7, }; /** @@ -901,6 +982,16 @@ enum drm_asahi_sync_type { /** @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ: Timeline sync object */ DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ = 1, + + /** + * @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE: Output-only queue + * timeline anchor, requiring CUMULATIVE_SIGNALS and a nonzero point. + * At most one is allowed. All outputs receive completion of this submit + * and the earlier anchor points. + * Other outputs must not alias the anchor. No execution dependency is + * added. Calls updating this private anchor must be externally serialized. + */ + DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE = 2, }; /** diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 9ab7494251e28..56f9f84689804 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -65,9 +67,12 @@ #include #include #include +#include #include #include +#include #include +#include #include #include #include @@ -94,13 +99,18 @@ #include #include #include +#include #include #include #include +#include #include #include +#include +#include #include #include +#include #include #include #include @@ -114,6 +124,8 @@ #include #include #include +#include +#include #include #include #include @@ -134,6 +146,7 @@ const size_t RUST_CONST_HELPER_ARCH_SLAB_MINALIGN = ARCH_SLAB_MINALIGN; const size_t RUST_CONST_HELPER_ARCH_KMALLOC_MINALIGN = ARCH_KMALLOC_MINALIGN; const size_t RUST_CONST_HELPER_PAGE_SIZE = PAGE_SIZE; +const size_t RUST_CONST_HELPER_GENLMSG_DEFAULT_SIZE = GENLMSG_DEFAULT_SIZE; const gfp_t RUST_CONST_HELPER_GFP_ATOMIC = GFP_ATOMIC; const gfp_t RUST_CONST_HELPER_GFP_KERNEL = GFP_KERNEL; const gfp_t RUST_CONST_HELPER_GFP_KERNEL_ACCOUNT = GFP_KERNEL_ACCOUNT; @@ -144,6 +157,7 @@ const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; +const u32 RUST_CONST_HELPER_APPLE_RTKIT_NEEDS_COREANALYTICS = APPLE_RTKIT_NEEDS_COREANALYTICS; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; @@ -153,8 +167,17 @@ const snd_ctl_elem_iface_t BINDINGS_SNDRV_CTL_ELEM_IFACE_MIXER = SNDRV_CTL_ELEM_ const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; +const u32 BINDINGS_IIO_CHAN_INFO_SCALE = IIO_CHAN_INFO_SCALE; const u32 BINDINGS_IIO_ANGL = IIO_ANGL; const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; +const u32 BINDINGS_IIO_COLORTEMP = IIO_COLORTEMP; +const u32 BINDINGS_IIO_ACCEL = IIO_ACCEL; +const u32 BINDINGS_IIO_ANGL_VEL = IIO_ANGL_VEL; +const u32 BINDINGS_IIO_TIMESTAMP = IIO_TIMESTAMP; +const u32 BINDINGS_IIO_MOD_X = IIO_MOD_X; +const u32 BINDINGS_IIO_MOD_Y = IIO_MOD_Y; +const u32 BINDINGS_IIO_MOD_Z = IIO_MOD_Z; +const u32 BINDINGS_IIO_CHAN_INFO_SAMP_FREQ = IIO_CHAN_INFO_SAMP_FREQ; const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index e63b62b54a17d..3b76d329688db 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -207,7 +207,8 @@ impl UnregisteredDevice { fops: &Self::GEM_FOPS, }; - const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(T::MODULE); + const GEM_FOPS: bindings::file_operations = + drm::gem::create_fops(crate::module::this_module::().as_ptr()); /// Create a new `UnregisteredDevice` for a `drm::Driver`. /// @@ -251,7 +252,7 @@ impl UnregisteredDevice { // SAFETY: // - `raw_data` is a valid pointer to uninitialized memory. // - `raw_data` will not move until it is dropped. - unsafe { data.__pinned_init(raw_data) }.inspect_err(|_| { + unsafe { pin_init::raw_try_init(raw_data, data) }.inspect_err(|_| { // SAFETY: `__drm_dev_alloc()` was successful, hence `drm_dev` must be valid and the // refcount must be non-zero. unsafe { bindings::drm_dev_put(drm_dev) }; @@ -300,6 +301,26 @@ impl Device { self.dev.get() } + /// Reports that device execution is unavailable and the recovery method is unknown. + /// + /// May sleep: call in task context outside fence signalling and driver locks. + /// A registration guard keeps the parent bound throughout event delivery; a device + /// still being probed or already unplugged cannot deliver the event. + pub fn report_wedged(&self) -> Result { + // SAFETY: Every device context satisfies the weaker Normal context invariants. + let normal = unsafe { self.assume_ctx::() }; + let _registration = normal.registration_guard().ok_or(ENODEV)?; + let raw = self.as_raw(); + // SAFETY: The referenced device and its registered primary node remain live under + // the registration guard. A probe failure has already been rejected above. + if unsafe { (*raw).primary.is_null() } { + return Err(ENODEV); + } + // SAFETY: The device/node are live, 0 asks DRM to report an unknown recovery method, + // and a NULL task means no userspace task is attributed as guilty. + crate::error::to_result(unsafe { bindings::drm_dev_wedged_event(raw, 0, ptr::null_mut()) }) + } + /// # Safety /// /// `ptr` must be a valid pointer to a `struct device` embedded in `Self`. diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 88709a7f9b36d..81f0a90e4a517 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -44,6 +44,11 @@ pub trait DriverFile { fn memory_stats(&self) -> Option { None } + + /// Emit completed per-engine busy time in nanoseconds for this client's fdinfo. + /// Engine names must be stable, unique and contain no whitespace or colons. + /// The default emits no engine keys. See `Documentation/gpu/drm-usage-stats.rst`. + fn engine_stats(&self, _emit: impl FnMut(&crate::str::CStr, u64)) {} } /// An open DRM File. @@ -136,6 +141,18 @@ impl File { // `driver_priv` was set by `open_callback()`. The seq_file read holds a reference to the // file, so `postclose_callback()` cannot run concurrently. let file = unsafe { File::::from_raw(raw_file) }; + file.inner().engine_stats(|name, ns| { + // SAFETY: The DRM core owns printer throughout this callback; the format and + // name are NUL-terminated, and the vararg types match %s and %llu. + unsafe { + bindings::drm_printf( + printer, + c"drm-engine-%s:\t%llu ns\n".as_char_ptr(), + name.as_char_ptr(), + ns as core::ffi::c_ulonglong, + ) + }; + }); let Some(stats) = file.inner().memory_stats() else { return; }; @@ -164,7 +181,7 @@ impl File { // SAFETY: `printer` is the printer `drm_show_fdinfo()` passes, valid for this call; `raw` // and the region name outlive the call. unsafe { - bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_ptr()) + bindings::drm_print_memory_stats(printer, &raw, supported, c"system".as_char_ptr()) }; } From f4b1301d77c938917f81937882c1d731de1507c4 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:26:09 -0500 Subject: [PATCH 33/37] selftests: asahi: account for the paired-header fixture through its runner Signed-off-by: Satya Benson --- tools/asahi/run-host-tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/asahi/run-host-tests.py b/tools/asahi/run-host-tests.py index 53f803bddc8e5..97087990539df 100755 --- a/tools/asahi/run-host-tests.py +++ b/tools/asahi/run-host-tests.py @@ -186,6 +186,9 @@ def mutations(flag, names, base, expect=KILL): def jobs(work, conf, dtc): found = {p.name for p in ASAHI.iterdir() if re.search(r'-tests\.(py|rs|sh)$', p.name) and p.name != Path(__file__).name} + # This Rust fixture is compiled and executed by the paired-header wrapper. + if 'test-m3-uapi.py' in ASAHI_TESTS: + found.discard('m3-uapi-tests.rs') known = set(ASAHI_TESTS) | set(ASAHI_RUST) | set(ASAHI_SHELL) missing = sorted(found - known) gone = sorted(name for name in known if not (ASAHI / name).is_file()) From 95f3e2a637284e0ffc8b7a981acbe9bb2a7e3177 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 21:59:22 -0500 Subject: [PATCH 34/37] drm/asahi: use the 7.3 MMIO mapping transaction and ownership contract Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/mmu/m3.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs index a63101a813426..e9dc7184d6582 100644 --- a/drivers/gpu/drm/asahi/mmu/m3.rs +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -56,7 +56,7 @@ impl Vm { return Err(EINVAL); } // SAFETY: As in `map_io`, the exec lock is only used for this mapping. - let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + let mut vm_exec = unsafe { self.gpuvm.exec_lock_vm(None, false)? }; let inner = vm_exec.unique_gpuvm.data(); let payload = KernelMappingInner { owner: self.gpuvm.clone(), @@ -82,18 +82,35 @@ impl Vm { let iova = node.start(); let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; + let transaction_uat = inner.uat_inner.clone(); + let handoff = transaction_uat.lock_m4_upper(inner.is_kernel)?; let mutation = inner.mapping_mutation(); let ret = inner .page_table .map_pages(iova..end, phys as PhysicalAddr, prot, false); + let discard = if ret.is_err() { + inner.page_table.discard_partial_map(iova..end) + } else { + Ok(()) + }; + drop(handoff); if ret.is_err() { - inner.rollback_failed_map(iova, size as u64); + inner.finish_failed_map(iova, size); + } else { + inner.tlbi_contexts_naming_root(iova, size); + } + let retain = discard.is_err() && inner.uat_inner.m4_adopted; + if retain { + inner.uat_inner.failed.store(true, Ordering::Release); } - // Drop the exec lock before the node, as in `map_io`. drop(mutation); - let _ = inner; + drop(vm_exec); + if retain { + core::mem::forget(node); + return Err(ret.err().unwrap_or(EIO)); + } ret?; - Ok(KernelMapping(node)) + Ok(KernelMapping(ManuallyDrop::new(node))) } /// [`Vm::map_io`] at `iova`, without the per-mapping log line. From 4abcf6603441027e0fce2798a0fc4151569b8b20 Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 22:00:06 -0500 Subject: [PATCH 35/37] drm/asahi: expose adopted upper-root selection to the T8122 backend Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/mmu.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 437685bfad1de..55a9fb7de0c22 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -3140,7 +3140,7 @@ impl Uat { /// Creates the same owned-RAM adopted UAT, choosing whether application contexts also /// name the firmware upper root. M4 qualifies only context 0 for firmware-upper access; /// its application TTBAT upper words stay zero. G15 preserves its existing shared upper. - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_M3_T8122))] pub(crate) fn new_adopted_with_user_upper( dev: &driver::AsahiDrmDevice, geometry: UatGeometry, From 9f442fa11953d5b1af1bfb050dbad3806ea4639b Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Wed, 4 Nov 2026 22:47:00 -0500 Subject: [PATCH 36/37] drm/asahi: adapt T8122 resources and module pins to 7.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use lifetime-aware managed MMIO, the current coherent DMA address accessor and module metadata. Keep Aurora’s identity-selected hwdata block instead of restoring the older diagnostic opt-out. Signed-off-by: Satya Benson --- drivers/gpu/drm/asahi/m3/m3_adt_config.rs | 24 +++++++---------------- drivers/gpu/drm/asahi/m3/m3_device.rs | 7 +++---- drivers/gpu/drm/asahi/m3/m3_memory.rs | 4 ++-- drivers/gpu/drm/asahi/m3/m3_params.rs | 12 ------------ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 8 ++++---- 5 files changed, 16 insertions(+), 39 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs index 1893c113cf2a7..983917b3b5686 100644 --- a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs @@ -920,27 +920,17 @@ fn build_images( for &(at, value) in hwdata_words { write(&mut hwdata, at, &value.to_le_bytes())?; } - // T8122 with the G15 14.8.3 firmware (the only image the runtime accepts there): the 20 - // words of crate::m3::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. + // T8122's exact chip/firmware identity selects this block on Aurora; keep the + // existing constant model instead of restoring omacom's diagnostic opt-out. if crate::m3::t8122_start::is_t8122(soc) && core::ptr::eq(firmware.layout, &crate::m3::m3_firmware::T8122_LAYOUT) && cfg.gpu_gen == hw::GpuGen::G15 { - if m3_params::t8122_hwdata_418c() { - let a = hwdata - .get_mut(HWDATA_A..HWDATA_A + size_of::()) - .ok_or(EINVAL)?; - t8122_hwdata::write(a).map_err(|()| EINVAL)?; - dev_info!( - dev, - "M3 G15G: HwDataA +0x418c..+0x41db: wrote the 20 words (asahi.t8122_hwdata_418c=0 leaves them 0)\n" - ); - } else { - dev_info!( - dev, - "M3 G15G: HwDataA +0x418c..+0x41db left 0 (asahi.t8122_hwdata_418c=0)\n" - ); - } + let a = hwdata + .get_mut(HWDATA_A..HWDATA_A + size_of::()) + .ok_or(EINVAL)?; + t8122_hwdata::write(a).map_err(|()| EINVAL)?; + dev_info!(dev, "M3 G15G: wrote identity-selected HwDataA +0x418c block\n"); } // On T8122, log and require the two HwDataB words the firmware's power management depends // on: it powers the GPU cores up for a job only when +0xa38 and +0xa40 are both nonzero. diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index c43017a8933c0..caef9657b4c9f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -7,10 +7,9 @@ use kernel::{ bindings, c_str, device::Core, - devres::Devres, error::to_result, io::{ - mem::IoMem, + mem::DevresIoMem, Io, IoBase, }, @@ -28,8 +27,8 @@ const ASC_CPU_RUN: u32 = 1 << 4; pub(crate) struct Device { dev: ARef, - asc: Devres>, - sgx: Devres>, + asc: DevresIoMem<0x4000>, + sgx: DevresIoMem, firmware: Firmware, core_mask: u64, soc: &'static Soc, diff --git a/drivers/gpu/drm/asahi/m3/m3_memory.rs b/drivers/gpu/drm/asahi/m3/m3_memory.rs index 5e8018a802e70..6ba456e382e57 100644 --- a/drivers/gpu/drm/asahi/m3/m3_memory.rs +++ b/drivers/gpu/drm/asahi/m3/m3_memory.rs @@ -68,7 +68,7 @@ impl Buffer { // diagnostic; all mappings retire before normal DMA allocation Drop. let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_ref().as_bound()},allocated,GFP_KERNEL, kernel::dma::attrs::DMA_ATTR_NO_WARN).inspect_err(|_| dev_err!(dev.as_ref(),"M3: cannot allocate {} KiB of GPU memory\n",allocated/1024))?; - let pa:usize=object.dma_handle().try_into()?; + let pa:usize=object.dma_address().try_into()?; let mapping=if let Some(a)=address {fw.map_io_m3(a&!0x3fff,pa,allocated,protection)?} else {fw.map_io_in_range(0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000,pa,allocated,protection)?}; let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io_m3(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; @@ -100,7 +100,7 @@ impl Buffer { pub(crate) fn map_gpu_view(&mut self,vm:&mmu::Vm)->Result { let address=self.gpu_va()?&!0x3fff; match &mut self.object { - Backing::Coherent(object)=>vm.map_io_m3(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), + Backing::Coherent(object)=>vm.map_io_m3(address,object.dma_address().try_into()?,object.size(),self.gpu_protection), Backing::Paged(object)=>object.map_at(vm,address,self.gpu_protection,true), } } diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs index 8167c6c3ec05c..85033c88e4192 100644 --- a/drivers/gpu/drm/asahi/m3/m3_params.rs +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -665,18 +665,6 @@ m3_param!("t8122_unit_mask_b", T8122_UNIT_MASK_B, parse_t8122_number); static T8122_PSTATE_CAP: AtomicU64 = AtomicU64::new(T8122_UNSET); m3_param!("t8122_pstate_cap", T8122_PSTATE_CAP, parse_t8122_number); -/// `asahi.t8122_hwdata_418c=0|1`: on a T8122 with the G15 14.8.3 firmware, 1 (the default) -/// writes the 20 words at HwDataA +0x418c ([`crate::m3::t8122_hwdata`]); 0 leaves the -/// range zero, for an A/B test. Readable in sysfs. -static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::m3::t8122_hwdata::DEFAULT); -m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::m3::t8122_hwdata::parse, 0o444, - Some(get_atomic_param)); - -/// Whether `asahi.t8122_hwdata_418c` asks for the HwDataA +0x418c words (the default). -pub(crate) fn t8122_hwdata_418c() -> bool { - crate::m3::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) -} - /// The T8122 start-experiment parameters of this boot, as given. pub(crate) fn t8122_params() -> crate::m3::t8122_knobs::Raw { let get = |p: &AtomicU64| p.load(Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 1a366d7f4b340..f9dd341aa89fb 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -991,7 +991,7 @@ static FIRMWARE_START: policy::StartLatch = policy::StartLatch::new(); fn pin_module() -> bool { // SAFETY: THIS_MODULE is this module, whose code is running; try_module_get fails once // removal has begun instead of reviving the reference count. - unsafe { kernel::bindings::try_module_get(crate::THIS_MODULE.as_ptr()) } + unsafe { kernel::bindings::try_module_get(kernel::module::this_module::().as_ptr()) } } /// Without module unloading there is no removal to race. #[cfg(not(CONFIG_MODULE_UNLOAD))] @@ -1000,7 +1000,7 @@ fn pin_module() -> bool { true } #[cfg(CONFIG_MODULE_UNLOAD)] fn unpin_module() { // SAFETY: called only to drop a reference pin_module took. - unsafe { kernel::bindings::module_put(crate::THIS_MODULE.as_ptr()) }; + unsafe { kernel::bindings::module_put(kernel::module::this_module::().as_ptr()) }; } #[cfg(not(CONFIG_MODULE_UNLOAD))] fn unpin_module() {} @@ -1074,7 +1074,7 @@ impl Runtime { crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { - unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; + unsafe {kernel::bindings::__module_get(kernel::module::this_module::().as_ptr())}; core::mem::forget(transport);core::mem::forget(state); core::mem::forget(adopted); core::mem::forget(drm);core::mem::forget(device); @@ -1285,7 +1285,7 @@ impl Drop for Runtime { for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { // SAFETY: a live module reference pins callbacks with retained DMA. - unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; + unsafe { kernel::bindings::__module_get(kernel::module::this_module::().as_ptr()) }; return; } // SAFETY: no GPU command is pending and ASC is stopped. Field order From 6ca4d4fa5a339eb761ff967ed6b8858f720844a2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 7 Nov 2026 12:23:14 -0500 Subject: [PATCH 37/37] drm/asahi: initialize the M3 backend slot in the M5 and G18 probes The M5 and G18 probes came in with integration/aurora after this series gave AsahiPlatformDriverData an m3 field, so a kernel with CONFIG_DRM_ASAHI_M5 or CONFIG_DRM_ASAHI_G18 no longer builds. Neither probe starts an M3 backend: set the field to None. Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/driver.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index fd5bc2157a26b..24b9ac9ae2bea 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -535,6 +535,7 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + m3: None, #[cfg(CONFIG_DRM_ASAHI_G16)] g16: None, raw_dev: core::ptr::null_mut(), @@ -629,6 +630,7 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + m3: None, #[cfg(CONFIG_DRM_ASAHI_G16)] g16: None, raw_dev: core::ptr::null_mut(),